* [dpdk-dev] [PATCH v2] pipeline: add register array support to SWX pipeline @ 2021-02-26 20:26 Cristian Dumitrescu 2021-03-15 22:22 ` [dpdk-dev] [PATCH v3 1/2] " Cristian Dumitrescu 0 siblings, 1 reply; 8+ messages in thread From: Cristian Dumitrescu @ 2021-02-26 20:26 UTC (permalink / raw) To: dev Register arrays are stateful objects that can be read & modified by both the data plane and the control plane, as opposed to tables, which are read-only for data plane. One key use-case is the implementation of stats counters. Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com> --- examples/pipeline/cli.c | 125 ++ lib/librte_pipeline/rte_swx_ctl.h | 79 + lib/librte_pipeline/rte_swx_pipeline.c | 1712 +++++++++++++++++-- lib/librte_pipeline/rte_swx_pipeline.h | 21 + lib/librte_pipeline/rte_swx_pipeline_spec.c | 91 + lib/librte_pipeline/version.map | 6 + 6 files changed, 1855 insertions(+), 179 deletions(-) diff --git a/examples/pipeline/cli.c b/examples/pipeline/cli.c index 30c2dd34d..cdf8f13f3 100644 --- a/examples/pipeline/cli.c +++ b/examples/pipeline/cli.c @@ -1009,6 +1009,105 @@ cmd_pipeline_table_update(char **tokens, fclose(file_default); } +static const char cmd_pipeline_regrd_help[] = +"pipeline <pipeline_name> regrd <register_array_name> <index>\n"; + +static void +cmd_pipeline_regrd(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 5) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regrd")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regrd"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_read(p->p, name, idx, &value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } + + snprintf(out, out_size, "0x%" PRIx64 "\n", value); +} + +static const char cmd_pipeline_regwr_help[] = +"pipeline <pipeline_name> regwr <register_array_name> <index> <value>\n"; + +static void +cmd_pipeline_regwr(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 6) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regwr")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regwr"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + if (parser_read_uint64(&value, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "value"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_write(p->p, name, idx, value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + static const char cmd_pipeline_stats_help[] = "pipeline <pipeline_name> stats\n"; @@ -1202,6 +1301,8 @@ cmd_help(char **tokens, "\tpipeline port out\n" "\tpipeline build\n" "\tpipeline table update\n" + "\tpipeline regrd\n" + "\tpipeline regwr\n" "\tpipeline stats\n" "\tthread pipeline enable\n" "\tthread pipeline disable\n\n"); @@ -1254,6 +1355,18 @@ cmd_help(char **tokens, return; } + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regrd") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regrd_help); + return; + } + + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regwr") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regwr_help); + return; + } + if ((strcmp(tokens[0], "pipeline") == 0) && (n_tokens == 2) && (strcmp(tokens[1], "stats") == 0)) { snprintf(out, out_size, "\n%s\n", cmd_pipeline_stats_help); @@ -1356,6 +1469,18 @@ cli_process(char *in, char *out, size_t out_size, void *obj) return; } + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regrd") == 0)) { + cmd_pipeline_regrd(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regwr") == 0)) { + cmd_pipeline_regwr(tokens, n_tokens, out, out_size, obj); + return; + } + if ((n_tokens >= 3) && (strcmp(tokens[2], "stats") == 0)) { cmd_pipeline_stats(tokens, n_tokens, out, out_size, diff --git a/lib/librte_pipeline/rte_swx_ctl.h b/lib/librte_pipeline/rte_swx_ctl.h index 530671db1..c0a55c6f0 100644 --- a/lib/librte_pipeline/rte_swx_ctl.h +++ b/lib/librte_pipeline/rte_swx_ctl.h @@ -46,6 +46,9 @@ struct rte_swx_ctl_pipeline_info { /** Number of tables. */ uint32_t n_tables; + + /** Number of register arrays. */ + uint32_t n_regarrays; }; /** @@ -557,6 +560,82 @@ rte_swx_ctl_pipeline_table_fprintf(FILE *f, struct rte_swx_ctl_pipeline *ctl, const char *table_name); +/* + * Register Array Query API. + */ + +/** Register array info. */ +struct rte_swx_ctl_regarray_info { + /** Register array name. */ + char name[RTE_SWX_CTL_NAME_SIZE]; + + /** Register array size. */ + uint32_t size; +}; + +/** + * Register array info get + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_id + * Register array ID (0 .. *n_regarrays* - 1). + * @param[out] regarray + * Register array info. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray); + +/** + * Register read + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[out] value + * Current register value. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value); + +/** + * Register write + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[in] value + * Value to be written to the register. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value); + /** * Pipeline control free * diff --git a/lib/librte_pipeline/rte_swx_pipeline.c b/lib/librte_pipeline/rte_swx_pipeline.c index eaaed7a0a..580a3a84b 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.c +++ b/lib/librte_pipeline/rte_swx_pipeline.c @@ -361,6 +361,53 @@ enum instruction_type { INSTR_ALU_SHR_MI, /* dst = MEF, src = I */ INSTR_ALU_SHR_HI, /* dst = H, src = I */ + /* regprefetch REGARRAY index + * prefetch REGARRAY[index] + * index = HMEFTI + */ + INSTR_REGPREFETCH_RH, /* index = H */ + INSTR_REGPREFETCH_RM, /* index = MEFT */ + INSTR_REGPREFETCH_RI, /* index = I */ + + /* regrd dst REGARRAY index + * dst = REGARRAY[index] + * dst = HMEF, index = HMEFTI + */ + INSTR_REGRD_HRH, /* dst = H, index = H */ + INSTR_REGRD_HRM, /* dst = H, index = MEFT */ + INSTR_REGRD_HRI, /* dst = H, index = I */ + INSTR_REGRD_MRH, /* dst = MEF, index = H */ + INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */ + INSTR_REGRD_MRI, /* dst = MEF, index = I */ + + /* regwr REGARRAY index src + * REGARRAY[index] = src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGWR_RHH, /* index = H, src = H */ + INSTR_REGWR_RHM, /* index = H, src = MEFT */ + INSTR_REGWR_RHI, /* index = H, src = I */ + INSTR_REGWR_RMH, /* index = MEFT, src = H */ + INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGWR_RMI, /* index = MEFT, src = I */ + INSTR_REGWR_RIH, /* index = I, src = H */ + INSTR_REGWR_RIM, /* index = I, src = MEFT */ + INSTR_REGWR_RII, /* index = I, src = I */ + + /* regadd REGARRAY index src + * REGARRAY[index] += src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGADD_RHH, /* index = H, src = H */ + INSTR_REGADD_RHM, /* index = H, src = MEFT */ + INSTR_REGADD_RHI, /* index = H, src = I */ + INSTR_REGADD_RMH, /* index = MEFT, src = H */ + INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGADD_RMI, /* index = MEFT, src = I */ + INSTR_REGADD_RIH, /* index = I, src = H */ + INSTR_REGADD_RIM, /* index = I, src = MEFT */ + INSTR_REGADD_RII, /* index = I, src = I */ + /* table TABLE */ INSTR_TABLE, @@ -495,6 +542,21 @@ struct instr_dst_src { }; }; +struct instr_regarray { + uint8_t regarray_id; + uint8_t pad[3]; + + union { + struct instr_operand idx; + uint32_t idx_val; + }; + + union { + struct instr_operand dstsrc; + uint64_t dstsrc_val; + }; +}; + struct instr_dma { struct { uint8_t header_id[8]; @@ -529,6 +591,7 @@ struct instruction { struct instr_io io; struct instr_hdr_validity valid; struct instr_dst_src mov; + struct instr_regarray regarray; struct instr_dma dma; struct instr_dst_src alu; struct instr_table table; @@ -608,6 +671,23 @@ struct table_runtime { uint8_t **key; }; +/* + * Register array. + */ +struct regarray { + TAILQ_ENTRY(regarray) node; + char name[RTE_SWX_NAME_SIZE]; + uint32_t size; + uint32_t id; +}; + +TAILQ_HEAD(regarray_tailq, regarray); + +struct regarray_runtime { + uint64_t *regarray; + uint32_t size_mask; +}; + /* * Pipeline. */ @@ -983,11 +1063,13 @@ struct rte_swx_pipeline { struct action_tailq actions; struct table_type_tailq table_types; struct table_tailq tables; + struct regarray_tailq regarrays; struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; struct rte_swx_table_state *table_state; + struct regarray_runtime *regarray_runtime; struct instruction *instructions; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; @@ -998,6 +1080,7 @@ struct rte_swx_pipeline { uint32_t n_extern_funcs; uint32_t n_actions; uint32_t n_tables; + uint32_t n_regarrays; uint32_t n_headers; uint32_t thread_id; uint32_t port_id; @@ -4622,252 +4705,1278 @@ instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p) } /* - * jmp. + * regarray. */ -static struct action * -action_find(struct rte_swx_pipeline *p, const char *name); - -static int -instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(n_tokens == 2, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - instr->type = INSTR_JMP; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name); static int -instr_jmp_valid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regprefetch_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *regarray = tokens[1], *idx = tokens[2]; + struct regarray *r; + struct field *fidx; + uint32_t idx_struct_id, idx_val; CHECK(n_tokens == 3, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); + + /* REGPREFETCH_RH, REGPREFETCH_RM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGPREFETCH_RM; + if (idx[0] == 'h') + instr->type = INSTR_REGPREFETCH_RH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = 0; /* Unused. */ + return 0; + } - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + /* REGPREFETCH_RI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - instr->type = INSTR_JMP_VALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; + instr->type = INSTR_REGPREFETCH_RI; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = 0; /* Unused. */ return 0; } static int -instr_jmp_invalid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regrd_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3]; + struct regarray *r; + struct field *fdst, *fidx; + uint32_t dst_struct_id, idx_struct_id, idx_val; - CHECK(n_tokens == 3, EINVAL); + CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); - instr->type = INSTR_JMP_INVALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; - return 0; -} + /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGRD_MRM; + if (dst[0] == 'h' && idx[0] == 'm') + instr->type = INSTR_REGRD_HRM; + if (dst[0] == 'm' && idx[0] == 'h') + instr->type = INSTR_REGRD_MRH; + if (dst[0] == 'h' && idx[0] == 'h') + instr->type = INSTR_REGRD_HRH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; + return 0; + } -static int -instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + /* REGRD_MRI, REGRD_HRI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGRD_MRI; + if (dst[0] == 'h') + instr->type = INSTR_REGRD_HRI; - instr->type = INSTR_JMP_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; return 0; } static int -instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regwr_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; - strcpy(data->jmp_label, tokens[1]); + CHECK(n_tokens == 4, EINVAL); - instr->type = INSTR_JMP_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + r = regarray_find(p, regarray); + CHECK(r, EINVAL); -static int -instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGWR_RMM; + if (idx[0] == 'h' && src[0] == 'm') + instr->type = INSTR_REGWR_RHM; + if (idx[0] == 'm' && src[0] == 'h') + instr->type = INSTR_REGWR_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGWR_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + /* REGWR_RHI, REGWR_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGWR_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGWR_RHI; - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; + return 0; + } - instr->type = INSTR_JMP_ACTION_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + /* REGWR_RIH, REGWR_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); -static int -instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + instr->type = INSTR_REGWR_RIM; + if (idx[0] == 'h') + instr->type = INSTR_REGWR_RIH; - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - strcpy(data->jmp_label, tokens[1]); + /* REGWR_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGWR_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; - instr->type = INSTR_JMP_ACTION_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; return 0; } static int -instr_jmp_eq_translate(struct rte_swx_pipeline *p, +instr_regadd_translate(struct rte_swx_pipeline *p, struct action *action, char **tokens, int n_tokens, struct instruction *instr, - struct instruction_data *data) + struct instruction_data *data __rte_unused) { - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGADD_RMM; + if (idx[0] == 'h' && src[0] == 'm') + instr->type = INSTR_REGADD_RHM; + if (idx[0] == 'm' && src[0] == 'h') + instr->type = INSTR_REGADD_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGADD_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - /* JMP_EQ or JMP_EQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_EQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_EQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + /* REGADD_RHI, REGADD_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; + instr->type = INSTR_REGADD_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGADD_RHI; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; return 0; } - /* JMP_EQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* REGADD_RIH, REGADD_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - if (a[0] == 'h') - b_val = hton64(b_val) >> (64 - fa->n_bits); + instr->type = INSTR_REGADD_RIM; + if (idx[0] == 'h') + instr->type = INSTR_REGADD_RIH; - instr->type = INSTR_JMP_EQ_I; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b_val = b_val; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } + + /* REGADD_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); + + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGADD_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; return 0; } -static int -instr_jmp_neq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; +#define REGPREFETCH_RM(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + rte_prefetch0(®array[idx]); \ +} - CHECK(n_tokens == 4, EINVAL); +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN - strcpy(data->jmp_label, tokens[1]); +#define REGPREFETCH_RH(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + rte_prefetch0(®array[idx]); \ +} - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); +#else - /* JMP_NEQ or JMP_NEQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_NEQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_NEQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ +#define REGPREFETCH_RH REGPREFETCH_RM - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; - return 0; - } +#endif - /* JMP_NEQ_I. */ - b_val = strtoull(b, &b, 0); +#define REGPREFETCH_RI(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + rte_prefetch0(®array[idx]); \ +} + +static inline void +instr_regprefetch_rh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); + + /* Structs. */ + REGPREFETCH_RH(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_rm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); + + /* Structs. */ + REGPREFETCH_RM(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_ri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); + + /* Structs. */ + REGPREFETCH_RI(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +#define REGRD_MRM(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (regarray[idx] & dstsrc64_mask); \ +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define REGRD_MRH(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (regarray[idx] & dstsrc64_mask); \ +} + +#define REGRD_HRM(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (hton64(regarray[idx]) & dstsrc64_mask); \ +} + +#define REGRD_HRH(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (hton64(regarray[idx]) & dstsrc64_mask); \ +} + +#else + +#define REGRD_MRH REGRD_MRM +#define REGRD_HRM REGRD_MRM +#define REGRD_HRH REGRD_MRM + +#endif + +#define REGRD_MRI(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (regarray[idx] & dstsrc64_mask); \ +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define REGRD_HRI(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (hton64(regarray[idx]) & dstsrc64_mask); \ +} + +#else + +#define REGRD_HRI REGRD_MRI + +#endif + +static inline void +instr_regrd_hrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); + + /* Structs. */ + REGRD_HRH(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); + + /* Structs. */ + REGRD_HRM(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); + + /* Structs. */ + REGRD_MRH(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id); + + /* Structs. */ + REGRD_MRM(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); + + /* Structs. */ + REGRD_HRI(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); + + /* Structs. */ + REGRD_MRI(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +#define REGWR_RMM(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + uint64_t dstsrc = dstsrc64 & dstsrc64_mask; \ + \ + regarray[idx] operator dstsrc; \ +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define REGWR_RMH(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc = ntoh64(dstsrc64) >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + regarray[idx] operator dstsrc; \ +} + +#define REGWR_RHM(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + uint64_t dstsrc = dstsrc64 & dstsrc64_mask; \ + \ + regarray[idx] operator dstsrc; \ +} + +#define REGWR_RHH(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc = ntoh64(dstsrc64) >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + regarray[idx] operator dstsrc; \ +} + +#else + +#define REGWR_RMH REGWR_RMM +#define REGWR_RHM REGWR_RMM +#define REGWR_RHH REGWR_RMM + +#endif + +#define REGWR_RMI(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint64_t dstsrc = (ip)->regarray.dstsrc_val; \ + \ + regarray[idx] operator dstsrc; \ +} + +#define REGWR_RIM(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + uint64_t dstsrc = dstsrc64 & dstsrc64_mask; \ + \ + regarray[idx] operator dstsrc; \ +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define REGWR_RHI(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint64_t dstsrc = (ip)->regarray.dstsrc_val; \ + \ + regarray[idx] operator dstsrc; \ +} + +#define REGWR_RIH(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc = ntoh64(dstsrc64) >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + regarray[idx] operator dstsrc; \ +} + +#else + +#define REGWR_RHI REGWR_RMI +#define REGWR_RIH REGWR_RIM + +#endif + +#define REGWR_RII(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint64_t dstsrc = (ip)->regarray.dstsrc_val; \ + \ + regarray[idx] operator dstsrc; \ +} + +static inline void +instr_regwr_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); + + /* Structs. */ + REGWR_RHH(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); + + /* Structs. */ + REGWR_RHM(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); + + /* Structs. */ + REGWR_RMH(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); + + /* Structs. */ + REGWR_RMM(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); + + /* Structs. */ + REGWR_RHI(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); + + /* Structs. */ + REGWR_RMI(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); + + /* Structs. */ + REGWR_RIH(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); + + /* Structs. */ + REGWR_RIM(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); + + /* Structs. */ + REGWR_RII(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); + + /* Structs. */ + REGWR_RHH(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); + + /* Structs. */ + REGWR_RHM(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); + + /* Structs. */ + REGWR_RMH(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); + + /* Structs. */ + REGWR_RMM(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); + + /* Structs. */ + REGWR_RHI(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); + + /* Structs. */ + REGWR_RMI(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); + + /* Structs. */ + REGWR_RIH(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); + + /* Structs. */ + REGWR_RIM(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); + + /* Structs. */ + REGWR_RII(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +/* + * jmp. + */ +static struct action * +action_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_valid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_VALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_invalid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_INVALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_eq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_EQ or JMP_EQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_EQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_EQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_EQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = hton64(b_val) >> (64 - fa->n_bits); + + instr->type = INSTR_JMP_EQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = b_val; + return 0; +} + +static int +instr_jmp_neq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_NEQ or JMP_NEQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_NEQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_NEQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_NEQ_I. */ + b_val = strtoull(b, &b, 0); CHECK(!b[0], EINVAL); if (a[0] == 'h') @@ -5475,6 +6584,38 @@ instr_translate(struct rte_swx_pipeline *p, instr, data); + if (!strcmp(tokens[tpos], "regprefetch")) + return instr_regprefetch_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regrd")) + return instr_regrd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regwr")) + return instr_regwr_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regadd")) + return instr_regadd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + if (!strcmp(tokens[tpos], "table")) return instr_table_translate(p, action, @@ -6111,6 +7252,37 @@ static instr_exec_t instruction_table[] = { [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec, [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec, + [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec, + [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec, + [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec, + + [INSTR_REGRD_HRH] = instr_regrd_hrh_exec, + [INSTR_REGRD_HRM] = instr_regrd_hrm_exec, + [INSTR_REGRD_MRH] = instr_regrd_mrh_exec, + [INSTR_REGRD_MRM] = instr_regrd_mrm_exec, + [INSTR_REGRD_HRI] = instr_regrd_hri_exec, + [INSTR_REGRD_MRI] = instr_regrd_mri_exec, + + [INSTR_REGWR_RHH] = instr_regwr_rhh_exec, + [INSTR_REGWR_RHM] = instr_regwr_rhm_exec, + [INSTR_REGWR_RMH] = instr_regwr_rmh_exec, + [INSTR_REGWR_RMM] = instr_regwr_rmm_exec, + [INSTR_REGWR_RHI] = instr_regwr_rhi_exec, + [INSTR_REGWR_RMI] = instr_regwr_rmi_exec, + [INSTR_REGWR_RIH] = instr_regwr_rih_exec, + [INSTR_REGWR_RIM] = instr_regwr_rim_exec, + [INSTR_REGWR_RII] = instr_regwr_rii_exec, + + [INSTR_REGADD_RHH] = instr_regadd_rhh_exec, + [INSTR_REGADD_RHM] = instr_regadd_rhm_exec, + [INSTR_REGADD_RMH] = instr_regadd_rmh_exec, + [INSTR_REGADD_RMM] = instr_regadd_rmm_exec, + [INSTR_REGADD_RHI] = instr_regadd_rhi_exec, + [INSTR_REGADD_RMI] = instr_regadd_rmi_exec, + [INSTR_REGADD_RIH] = instr_regadd_rih_exec, + [INSTR_REGADD_RIM] = instr_regadd_rim_exec, + [INSTR_REGADD_RII] = instr_regadd_rii_exec, + [INSTR_TABLE] = instr_table_exec, [INSTR_EXTERN_OBJ] = instr_extern_obj_exec, [INSTR_EXTERN_FUNC] = instr_extern_func_exec, @@ -6823,6 +7995,119 @@ table_free(struct rte_swx_pipeline *p) } } +/* + * Register array. + */ +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name) +{ + struct regarray *elem; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct regarray * +regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id) +{ + struct regarray *elem = NULL; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (elem->id == id) + return elem; + + return NULL; +} + +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size) +{ + struct regarray *r; + + CHECK(p, EINVAL); + + CHECK_NAME(name, EINVAL); + CHECK(!regarray_find(p, name), EEXIST); + + CHECK(size, EINVAL); + size = rte_align32pow2(size); + + /* Memory allocation. */ + r = calloc(1, sizeof(struct regarray)); + CHECK(r, ENOMEM); + + /* Node initialization. */ + strcpy(r->name, name); + r->size = size; + r->id = p->n_regarrays; + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->regarrays, r, node); + p->n_regarrays++; + + return 0; +} + +static int +regarray_build(struct rte_swx_pipeline *p) +{ + struct regarray *regarray; + + p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime)); + CHECK(p->regarray_runtime, ENOMEM); + + TAILQ_FOREACH(regarray, &p->regarrays, node) { + struct regarray_runtime *r = &p->regarray_runtime[regarray->id]; + + r->regarray = calloc(regarray->size, sizeof(uint64_t)); + CHECK(r->regarray, ENOMEM); + + r->size_mask = regarray->size - 1; + } + + return 0; +} + +static void +regarray_build_free(struct rte_swx_pipeline *p) +{ + uint32_t i; + + if (!p->regarray_runtime) + return; + + for (i = 0; i < p->n_regarrays; i++) { + struct regarray_runtime *r = &p->regarray_runtime[i]; + + free(r->regarray); + } + + free(p->regarray_runtime); + p->regarray_runtime = NULL; +} + +static void +regarray_free(struct rte_swx_pipeline *p) +{ + regarray_build_free(p); + + for ( ; ; ) { + struct regarray *elem; + + elem = TAILQ_FIRST(&p->regarrays); + if (!elem) + break; + + TAILQ_REMOVE(&p->regarrays, elem, node); + free(elem); + } +} + /* * Pipeline. */ @@ -6851,6 +8136,7 @@ rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node) TAILQ_INIT(&pipeline->actions); TAILQ_INIT(&pipeline->table_types); TAILQ_INIT(&pipeline->tables); + TAILQ_INIT(&pipeline->regarrays); pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */ pipeline->numa_node = numa_node; @@ -6867,6 +8153,7 @@ rte_swx_pipeline_free(struct rte_swx_pipeline *p) free(p->instructions); + regarray_free(p); table_state_free(p); table_free(p); action_free(p); @@ -6951,10 +8238,15 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p) if (status) goto error; + status = regarray_build(p); + if (status) + goto error; + p->build_done = 1; return 0; error: + regarray_build_free(p); table_state_build_free(p); table_build_free(p); action_build_free(p); @@ -7015,6 +8307,7 @@ rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p, pipeline->n_ports_out = p->n_ports_out; pipeline->n_actions = n_actions; pipeline->n_tables = n_tables; + pipeline->n_regarrays = p->n_regarrays; return 0; } @@ -7222,3 +8515,64 @@ rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p, port->type->ops.stats_read(port->obj, stats); return 0; } + +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray) +{ + struct regarray *r; + + if (!p || !regarray) + return -EINVAL; + + r = regarray_find_by_id(p, regarray_id); + if (!r) + return -EINVAL; + + strcpy(regarray->name, r->name); + regarray->size = r->size; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name || !value) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + *value = r->regarray[regarray_index]; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + r->regarray[regarray_index] = value; + return 0; +} diff --git a/lib/librte_pipeline/rte_swx_pipeline.h b/lib/librte_pipeline/rte_swx_pipeline.h index f0a2cef77..c6e4e50a8 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.h +++ b/lib/librte_pipeline/rte_swx_pipeline.h @@ -616,6 +616,27 @@ rte_swx_pipeline_table_config(struct rte_swx_pipeline *p, const char *args, uint32_t size); +/** + * Pipeline register array configure + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Register array name. + * @param[in] size + * Number of registers in the array. Each register is 64-bit in size. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Register array with this name already exists. + */ +__rte_experimental +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size); + /** * Pipeline instructions configure * diff --git a/lib/librte_pipeline/rte_swx_pipeline_spec.c b/lib/librte_pipeline/rte_swx_pipeline_spec.c index f7884491b..31dcfc4c3 100644 --- a/lib/librte_pipeline/rte_swx_pipeline_spec.c +++ b/lib/librte_pipeline/rte_swx_pipeline_spec.c @@ -940,6 +940,68 @@ table_block_parse(struct table_spec *s, return -EINVAL; } +/* + * regarray. + * + * regarray NAME size SIZE + */ +struct regarray_spec { + char *name; + uint32_t size; +}; + +static void +regarray_spec_free(struct regarray_spec *s) +{ + if (!s) + return; + + free(s->name); + s->name = NULL; +} + +static int +regarray_statement_parse(struct regarray_spec *s, + char **tokens, + uint32_t n_tokens, + uint32_t n_lines, + uint32_t *err_line, + const char **err_msg) +{ + char *p; + + /* Check format. */ + if ((n_tokens != 4) || strcmp(tokens[2], "size")) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid regarray statement."; + return -EINVAL; + } + + /* spec. */ + s->name = strdup(tokens[1]); + if (!s->name) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Memory allocation failed."; + return -ENOMEM; + } + + p = tokens[3]; + s->size = strtoul(p, &p, 0); + if (p[0] || !s->size) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid size argument."; + return -EINVAL; + } + + return 0; +} + /* * apply. * @@ -1066,6 +1128,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, struct metadata_spec metadata_spec = {0}; struct action_spec action_spec = {0}; struct table_spec table_spec = {0}; + struct regarray_spec regarray_spec = {0}; struct apply_spec apply_spec = {0}; uint32_t n_lines; uint32_t block_mask = 0; @@ -1405,6 +1468,33 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, continue; } + /* regarray. */ + if (!strcmp(tokens[0], "regarray")) { + status = regarray_statement_parse(®array_spec, + tokens, + n_tokens, + n_lines, + err_line, + err_msg); + if (status) + goto error; + + status = rte_swx_pipeline_regarray_config(p, + regarray_spec.name, + regarray_spec.size); + if (status) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Register array configuration error."; + goto error; + } + + regarray_spec_free(®array_spec); + + continue; + } + /* apply. */ if (!strcmp(tokens[0], "apply")) { status = apply_statement_parse(&block_mask, @@ -1457,6 +1547,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, metadata_spec_free(&metadata_spec); action_spec_free(&action_spec); table_spec_free(&table_spec); + regarray_spec_free(®array_spec); apply_spec_free(&apply_spec); return status; } diff --git a/lib/librte_pipeline/version.map b/lib/librte_pipeline/version.map index 87c826f1b..26a9edf47 100644 --- a/lib/librte_pipeline/version.map +++ b/lib/librte_pipeline/version.map @@ -102,4 +102,10 @@ EXPERIMENTAL { rte_swx_pipeline_table_state_get; rte_swx_pipeline_table_state_set; rte_swx_pipeline_table_type_register; + + #added in 21.05 + rte_swx_ctl_pipeline_regarray_read; + rte_swx_ctl_pipeline_regarray_write; + rte_swx_ctl_regarray_info_get; + rte_swx_pipeline_regarray_config; }; -- 2.17.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* [dpdk-dev] [PATCH v3 1/2] pipeline: add register array support to SWX pipeline 2021-02-26 20:26 [dpdk-dev] [PATCH v2] pipeline: add register array support to SWX pipeline Cristian Dumitrescu @ 2021-03-15 22:22 ` Cristian Dumitrescu 2021-03-15 22:22 ` [dpdk-dev] [PATCH v3 2/2] pipeline: add meter support to the " Cristian Dumitrescu 2021-03-15 22:48 ` [dpdk-dev] [PATCH v4 1/2] pipeline: add register array support to " Cristian Dumitrescu 0 siblings, 2 replies; 8+ messages in thread From: Cristian Dumitrescu @ 2021-03-15 22:22 UTC (permalink / raw) To: dev Register arrays are stateful objects that can be read & modified by both the data plane and the control plane, as opposed to tables, which are read-only for data plane. One key use-case is the implementation of stats counters. Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com> --- examples/pipeline/cli.c | 125 ++ examples/pipeline/examples/registers.cli | 28 + examples/pipeline/examples/registers.spec | 67 + lib/librte_pipeline/rte_swx_ctl.h | 79 + lib/librte_pipeline/rte_swx_pipeline.c | 1602 +++++++++++++++++-- lib/librte_pipeline/rte_swx_pipeline.h | 24 + lib/librte_pipeline/rte_swx_pipeline_spec.c | 105 ++ lib/librte_pipeline/version.map | 6 + 8 files changed, 1882 insertions(+), 154 deletions(-) create mode 100644 examples/pipeline/examples/registers.cli create mode 100644 examples/pipeline/examples/registers.spec diff --git a/examples/pipeline/cli.c b/examples/pipeline/cli.c index 30c2dd34d..cdf8f13f3 100644 --- a/examples/pipeline/cli.c +++ b/examples/pipeline/cli.c @@ -1009,6 +1009,105 @@ cmd_pipeline_table_update(char **tokens, fclose(file_default); } +static const char cmd_pipeline_regrd_help[] = +"pipeline <pipeline_name> regrd <register_array_name> <index>\n"; + +static void +cmd_pipeline_regrd(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 5) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regrd")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regrd"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_read(p->p, name, idx, &value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } + + snprintf(out, out_size, "0x%" PRIx64 "\n", value); +} + +static const char cmd_pipeline_regwr_help[] = +"pipeline <pipeline_name> regwr <register_array_name> <index> <value>\n"; + +static void +cmd_pipeline_regwr(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 6) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regwr")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regwr"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + if (parser_read_uint64(&value, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "value"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_write(p->p, name, idx, value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + static const char cmd_pipeline_stats_help[] = "pipeline <pipeline_name> stats\n"; @@ -1202,6 +1301,8 @@ cmd_help(char **tokens, "\tpipeline port out\n" "\tpipeline build\n" "\tpipeline table update\n" + "\tpipeline regrd\n" + "\tpipeline regwr\n" "\tpipeline stats\n" "\tthread pipeline enable\n" "\tthread pipeline disable\n\n"); @@ -1254,6 +1355,18 @@ cmd_help(char **tokens, return; } + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regrd") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regrd_help); + return; + } + + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regwr") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regwr_help); + return; + } + if ((strcmp(tokens[0], "pipeline") == 0) && (n_tokens == 2) && (strcmp(tokens[1], "stats") == 0)) { snprintf(out, out_size, "\n%s\n", cmd_pipeline_stats_help); @@ -1356,6 +1469,18 @@ cli_process(char *in, char *out, size_t out_size, void *obj) return; } + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regrd") == 0)) { + cmd_pipeline_regrd(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regwr") == 0)) { + cmd_pipeline_regwr(tokens, n_tokens, out, out_size, obj); + return; + } + if ((n_tokens >= 3) && (strcmp(tokens[2], "stats") == 0)) { cmd_pipeline_stats(tokens, n_tokens, out, out_size, diff --git a/examples/pipeline/examples/registers.cli b/examples/pipeline/examples/registers.cli new file mode 100644 index 000000000..8d026294c --- /dev/null +++ b/examples/pipeline/examples/registers.cli @@ -0,0 +1,28 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2020 Intel Corporation + +; Example command line: +; ./build/examples/dpdk-pipeline -l0-1 -- -s ./examples/pipeline/examples/registers.cli + +mempool MEMPOOL0 buffer 2304 pool 32K cache 256 cpu 0 + +link LINK0 dev 0000:18:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK1 dev 0000:18:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK2 dev 0000:3b:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK3 dev 0000:3b:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on + +pipeline PIPELINE0 create 0 + +pipeline PIPELINE0 port in 0 link LINK0 rxq 0 bsz 32 +pipeline PIPELINE0 port in 1 link LINK1 rxq 0 bsz 32 +pipeline PIPELINE0 port in 2 link LINK2 rxq 0 bsz 32 +pipeline PIPELINE0 port in 3 link LINK3 rxq 0 bsz 32 + +pipeline PIPELINE0 port out 0 link LINK0 txq 0 bsz 32 +pipeline PIPELINE0 port out 1 link LINK1 txq 0 bsz 32 +pipeline PIPELINE0 port out 2 link LINK2 txq 0 bsz 32 +pipeline PIPELINE0 port out 3 link LINK3 txq 0 bsz 32 + +pipeline PIPELINE0 build ./examples/pipeline/examples/registers.spec + +thread 1 pipeline PIPELINE0 enable diff --git a/examples/pipeline/examples/registers.spec b/examples/pipeline/examples/registers.spec new file mode 100644 index 000000000..74a014ad0 --- /dev/null +++ b/examples/pipeline/examples/registers.spec @@ -0,0 +1,67 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2021 Intel Corporation + +; This program is setting up two register arrays called "pkt_counters" and "byte_counters". +; On every input packet (Ethernet/IPv4), the "pkt_counters" register at location indexed by +; the IPv4 header "Source Address" field is incremented, while the same location in the +; "byte_counters" array accummulates the value of the IPv4 header "Total Length" field. +; +; The "regrd" and "regwr" CLI commands can be used to read and write the current value of +; any register array location. + +// +// Headers. +// +struct ethernet_h { + bit<48> dst_addr + bit<48> src_addr + bit<16> ethertype +} + +struct ipv4_h { + bit<8> ver_ihl + bit<8> diffserv + bit<16> total_len + bit<16> identification + bit<16> flags_offset + bit<8> ttl + bit<8> protocol + bit<16> hdr_checksum + bit<32> src_addr + bit<32> dst_addr +} + +header ethernet instanceof ethernet_h +header ipv4 instanceof ipv4_h + +// +// Meta-data. +// +struct metadata_t { + bit<32> port_in + bit<32> port_out +} + +metadata instanceof metadata_t + +// +// Registers. +// +regarray pkt_counters size 65536 initval 0 +regarray byte_counters size 65536 initval 0 + +// +// Pipeline. +// +apply { + rx m.port_in + extract h.ethernet + extract h.ipv4 + regadd pkt_counters h.ipv4.src_addr 1 + regadd byte_counters h.ipv4.src_addr h.ipv4.total_len + mov m.port_out m.port_in + xor m.port_out 1 + emit h.ethernet + emit h.ipv4 + tx m.port_out +} diff --git a/lib/librte_pipeline/rte_swx_ctl.h b/lib/librte_pipeline/rte_swx_ctl.h index 530671db1..c0a55c6f0 100644 --- a/lib/librte_pipeline/rte_swx_ctl.h +++ b/lib/librte_pipeline/rte_swx_ctl.h @@ -46,6 +46,9 @@ struct rte_swx_ctl_pipeline_info { /** Number of tables. */ uint32_t n_tables; + + /** Number of register arrays. */ + uint32_t n_regarrays; }; /** @@ -557,6 +560,82 @@ rte_swx_ctl_pipeline_table_fprintf(FILE *f, struct rte_swx_ctl_pipeline *ctl, const char *table_name); +/* + * Register Array Query API. + */ + +/** Register array info. */ +struct rte_swx_ctl_regarray_info { + /** Register array name. */ + char name[RTE_SWX_CTL_NAME_SIZE]; + + /** Register array size. */ + uint32_t size; +}; + +/** + * Register array info get + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_id + * Register array ID (0 .. *n_regarrays* - 1). + * @param[out] regarray + * Register array info. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray); + +/** + * Register read + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[out] value + * Current register value. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value); + +/** + * Register write + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[in] value + * Value to be written to the register. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value); + /** * Pipeline control free * diff --git a/lib/librte_pipeline/rte_swx_pipeline.c b/lib/librte_pipeline/rte_swx_pipeline.c index eaaed7a0a..6d4a407ce 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.c +++ b/lib/librte_pipeline/rte_swx_pipeline.c @@ -45,9 +45,59 @@ do { \ #define TRACE(...) #endif +/* + * Environment. + */ #define ntoh64(x) rte_be_to_cpu_64(x) #define hton64(x) rte_cpu_to_be_64(x) +#ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE + +#include <rte_malloc.h> + +static void * +env_malloc(size_t size, size_t alignment, int numa_node) +{ + return rte_zmalloc_socket(NULL, size, alignment, numa_node); +} + +static void +env_free(void *start, size_t size __rte_unused) +{ + rte_free(start); +} + +#else + +#include <numa.h> + +static void * +env_malloc(size_t size, size_t alignment __rte_unused, int numa_node) +{ + void *start; + + if (numa_available() == -1) + return NULL; + + start = numa_alloc_onnode(size, numa_node); + if (!start) + return NULL; + + memset(start, 0, size); + return start; +} + +static void +env_free(void *start, size_t size) +{ + if (numa_available() == -1) + return; + + numa_free(start, size); +} + +#endif + /* * Struct. */ @@ -361,6 +411,53 @@ enum instruction_type { INSTR_ALU_SHR_MI, /* dst = MEF, src = I */ INSTR_ALU_SHR_HI, /* dst = H, src = I */ + /* regprefetch REGARRAY index + * prefetch REGARRAY[index] + * index = HMEFTI + */ + INSTR_REGPREFETCH_RH, /* index = H */ + INSTR_REGPREFETCH_RM, /* index = MEFT */ + INSTR_REGPREFETCH_RI, /* index = I */ + + /* regrd dst REGARRAY index + * dst = REGARRAY[index] + * dst = HMEF, index = HMEFTI + */ + INSTR_REGRD_HRH, /* dst = H, index = H */ + INSTR_REGRD_HRM, /* dst = H, index = MEFT */ + INSTR_REGRD_HRI, /* dst = H, index = I */ + INSTR_REGRD_MRH, /* dst = MEF, index = H */ + INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */ + INSTR_REGRD_MRI, /* dst = MEF, index = I */ + + /* regwr REGARRAY index src + * REGARRAY[index] = src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGWR_RHH, /* index = H, src = H */ + INSTR_REGWR_RHM, /* index = H, src = MEFT */ + INSTR_REGWR_RHI, /* index = H, src = I */ + INSTR_REGWR_RMH, /* index = MEFT, src = H */ + INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGWR_RMI, /* index = MEFT, src = I */ + INSTR_REGWR_RIH, /* index = I, src = H */ + INSTR_REGWR_RIM, /* index = I, src = MEFT */ + INSTR_REGWR_RII, /* index = I, src = I */ + + /* regadd REGARRAY index src + * REGARRAY[index] += src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGADD_RHH, /* index = H, src = H */ + INSTR_REGADD_RHM, /* index = H, src = MEFT */ + INSTR_REGADD_RHI, /* index = H, src = I */ + INSTR_REGADD_RMH, /* index = MEFT, src = H */ + INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGADD_RMI, /* index = MEFT, src = I */ + INSTR_REGADD_RIH, /* index = I, src = H */ + INSTR_REGADD_RIM, /* index = I, src = MEFT */ + INSTR_REGADD_RII, /* index = I, src = I */ + /* table TABLE */ INSTR_TABLE, @@ -495,6 +592,21 @@ struct instr_dst_src { }; }; +struct instr_regarray { + uint8_t regarray_id; + uint8_t pad[3]; + + union { + struct instr_operand idx; + uint32_t idx_val; + }; + + union { + struct instr_operand dstsrc; + uint64_t dstsrc_val; + }; +}; + struct instr_dma { struct { uint8_t header_id[8]; @@ -529,6 +641,7 @@ struct instruction { struct instr_io io; struct instr_hdr_validity valid; struct instr_dst_src mov; + struct instr_regarray regarray; struct instr_dma dma; struct instr_dst_src alu; struct instr_table table; @@ -608,6 +721,24 @@ struct table_runtime { uint8_t **key; }; +/* + * Register array. + */ +struct regarray { + TAILQ_ENTRY(regarray) node; + char name[RTE_SWX_NAME_SIZE]; + uint64_t init_val; + uint32_t size; + uint32_t id; +}; + +TAILQ_HEAD(regarray_tailq, regarray); + +struct regarray_runtime { + uint64_t *regarray; + uint32_t size_mask; +}; + /* * Pipeline. */ @@ -983,11 +1114,13 @@ struct rte_swx_pipeline { struct action_tailq actions; struct table_type_tailq table_types; struct table_tailq tables; + struct regarray_tailq regarrays; struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; struct rte_swx_table_state *table_state; + struct regarray_runtime *regarray_runtime; struct instruction *instructions; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; @@ -998,6 +1131,7 @@ struct rte_swx_pipeline { uint32_t n_extern_funcs; uint32_t n_actions; uint32_t n_tables; + uint32_t n_regarrays; uint32_t n_headers; uint32_t thread_id; uint32_t port_id; @@ -4622,222 +4756,1124 @@ instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p) } /* - * jmp. + * Register array. */ -static struct action * -action_find(struct rte_swx_pipeline *p, const char *name); - -static int -instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(n_tokens == 2, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - instr->type = INSTR_JMP; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name); static int -instr_jmp_valid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regprefetch_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *regarray = tokens[1], *idx = tokens[2]; + struct regarray *r; + struct field *fidx; + uint32_t idx_struct_id, idx_val; CHECK(n_tokens == 3, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); + + /* REGPREFETCH_RH, REGPREFETCH_RM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGPREFETCH_RM; + if (idx[0] == 'h') + instr->type = INSTR_REGPREFETCH_RH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = 0; /* Unused. */ + return 0; + } - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + /* REGPREFETCH_RI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - instr->type = INSTR_JMP_VALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; + instr->type = INSTR_REGPREFETCH_RI; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = 0; /* Unused. */ return 0; } static int -instr_jmp_invalid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regrd_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3]; + struct regarray *r; + struct field *fdst, *fidx; + uint32_t dst_struct_id, idx_struct_id, idx_val; - CHECK(n_tokens == 3, EINVAL); + CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); - instr->type = INSTR_JMP_INVALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; - return 0; -} + /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGRD_MRM; + if (dst[0] == 'h' && idx[0] != 'h') + instr->type = INSTR_REGRD_HRM; + if (dst[0] != 'h' && idx[0] == 'h') + instr->type = INSTR_REGRD_MRH; + if (dst[0] == 'h' && idx[0] == 'h') + instr->type = INSTR_REGRD_HRH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; + return 0; + } -static int -instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + /* REGRD_MRI, REGRD_HRI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGRD_MRI; + if (dst[0] == 'h') + instr->type = INSTR_REGRD_HRI; - instr->type = INSTR_JMP_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; return 0; } static int -instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regwr_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; - strcpy(data->jmp_label, tokens[1]); + CHECK(n_tokens == 4, EINVAL); - instr->type = INSTR_JMP_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + r = regarray_find(p, regarray); + CHECK(r, EINVAL); -static int -instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGWR_RMM; + if (idx[0] == 'h' && src[0] != 'h') + instr->type = INSTR_REGWR_RHM; + if (idx[0] != 'h' && src[0] == 'h') + instr->type = INSTR_REGWR_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGWR_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + /* REGWR_RHI, REGWR_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGWR_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGWR_RHI; - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; + return 0; + } - instr->type = INSTR_JMP_ACTION_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + /* REGWR_RIH, REGWR_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); -static int -instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + instr->type = INSTR_REGWR_RIM; + if (src[0] == 'h') + instr->type = INSTR_REGWR_RIH; - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - strcpy(data->jmp_label, tokens[1]); + /* REGWR_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGWR_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; - instr->type = INSTR_JMP_ACTION_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; return 0; } static int -instr_jmp_eq_translate(struct rte_swx_pipeline *p, +instr_regadd_translate(struct rte_swx_pipeline *p, struct action *action, char **tokens, int n_tokens, struct instruction *instr, - struct instruction_data *data) + struct instruction_data *data __rte_unused) { - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGADD_RMM; + if (idx[0] == 'h' && src[0] != 'h') + instr->type = INSTR_REGADD_RHM; + if (idx[0] != 'h' && src[0] == 'h') + instr->type = INSTR_REGADD_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGADD_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - /* JMP_EQ or JMP_EQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_EQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_EQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + /* REGADD_RHI, REGADD_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; + instr->type = INSTR_REGADD_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGADD_RHI; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; return 0; } - /* JMP_EQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* REGADD_RIH, REGADD_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - if (a[0] == 'h') - b_val = hton64(b_val) >> (64 - fa->n_bits); + instr->type = INSTR_REGADD_RIM; + if (src[0] == 'h') + instr->type = INSTR_REGADD_RIH; - instr->type = INSTR_JMP_EQ_I; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b_val = b_val; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } + + /* REGADD_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); + + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGADD_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; return 0; } -static int -instr_jmp_neq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +static inline uint64_t * +instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip) { - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + return r->regarray; +} + +static inline uint64_t +instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return idx; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; + + return idx; +} + +#else + +#define instr_regarray_idx_nbo instr_regarray_idx_hbo + +#endif + +static inline uint64_t +instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint64_t idx = ip->regarray.idx_val & r->size_mask; + + return idx; +} + +static inline uint64_t +instr_regarray_src_hbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + uint64_t src = src64 & src64_mask; + + return src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_src_nbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); + + return src; +} + +#else + +#define instr_regarray_src_nbo instr_regarray_src_hbo + +#endif + +static inline void +instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); + +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline void +instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +#else + +#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set + +#endif + +static inline void +instr_regprefetch_rh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + rte_prefetch0(®array[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_rm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + rte_prefetch0(®array[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_ri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + rte_prefetch0(®array[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t,ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +/* + * jmp. + */ +static struct action * +action_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_valid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_VALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_invalid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_INVALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_eq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_EQ or JMP_EQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_EQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_EQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_EQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = hton64(b_val) >> (64 - fa->n_bits); + + instr->type = INSTR_JMP_EQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = b_val; + return 0; +} + +static int +instr_jmp_neq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; uint64_t b_val; uint32_t a_struct_id, b_struct_id; @@ -5475,6 +6511,38 @@ instr_translate(struct rte_swx_pipeline *p, instr, data); + if (!strcmp(tokens[tpos], "regprefetch")) + return instr_regprefetch_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regrd")) + return instr_regrd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regwr")) + return instr_regwr_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regadd")) + return instr_regadd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + if (!strcmp(tokens[tpos], "table")) return instr_table_translate(p, action, @@ -6111,6 +7179,37 @@ static instr_exec_t instruction_table[] = { [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec, [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec, + [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec, + [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec, + [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec, + + [INSTR_REGRD_HRH] = instr_regrd_hrh_exec, + [INSTR_REGRD_HRM] = instr_regrd_hrm_exec, + [INSTR_REGRD_MRH] = instr_regrd_mrh_exec, + [INSTR_REGRD_MRM] = instr_regrd_mrm_exec, + [INSTR_REGRD_HRI] = instr_regrd_hri_exec, + [INSTR_REGRD_MRI] = instr_regrd_mri_exec, + + [INSTR_REGWR_RHH] = instr_regwr_rhh_exec, + [INSTR_REGWR_RHM] = instr_regwr_rhm_exec, + [INSTR_REGWR_RMH] = instr_regwr_rmh_exec, + [INSTR_REGWR_RMM] = instr_regwr_rmm_exec, + [INSTR_REGWR_RHI] = instr_regwr_rhi_exec, + [INSTR_REGWR_RMI] = instr_regwr_rmi_exec, + [INSTR_REGWR_RIH] = instr_regwr_rih_exec, + [INSTR_REGWR_RIM] = instr_regwr_rim_exec, + [INSTR_REGWR_RII] = instr_regwr_rii_exec, + + [INSTR_REGADD_RHH] = instr_regadd_rhh_exec, + [INSTR_REGADD_RHM] = instr_regadd_rhm_exec, + [INSTR_REGADD_RMH] = instr_regadd_rmh_exec, + [INSTR_REGADD_RMM] = instr_regadd_rmm_exec, + [INSTR_REGADD_RHI] = instr_regadd_rhi_exec, + [INSTR_REGADD_RMI] = instr_regadd_rmi_exec, + [INSTR_REGADD_RIH] = instr_regadd_rih_exec, + [INSTR_REGADD_RIM] = instr_regadd_rim_exec, + [INSTR_REGADD_RII] = instr_regadd_rii_exec, + [INSTR_TABLE] = instr_table_exec, [INSTR_EXTERN_OBJ] = instr_extern_obj_exec, [INSTR_EXTERN_FUNC] = instr_extern_func_exec, @@ -6823,6 +7922,132 @@ table_free(struct rte_swx_pipeline *p) } } +/* + * Register array. + */ +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name) +{ + struct regarray *elem; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct regarray * +regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id) +{ + struct regarray *elem = NULL; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (elem->id == id) + return elem; + + return NULL; +} + +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size, + uint64_t init_val) +{ + struct regarray *r; + + CHECK(p, EINVAL); + + CHECK_NAME(name, EINVAL); + CHECK(!regarray_find(p, name), EEXIST); + + CHECK(size, EINVAL); + size = rte_align32pow2(size); + + /* Memory allocation. */ + r = calloc(1, sizeof(struct regarray)); + CHECK(r, ENOMEM); + + /* Node initialization. */ + strcpy(r->name, name); + r->init_val = init_val; + r->size = size; + r->id = p->n_regarrays; + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->regarrays, r, node); + p->n_regarrays++; + + return 0; +} + +static int +regarray_build(struct rte_swx_pipeline *p) +{ + struct regarray *regarray; + + if (!p->n_regarrays) + return 0; + + p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime)); + CHECK(p->regarray_runtime, ENOMEM); + + TAILQ_FOREACH(regarray, &p->regarrays, node) { + struct regarray_runtime *r = &p->regarray_runtime[regarray->id]; + uint32_t i; + + r->regarray = env_malloc(regarray->size * sizeof(uint64_t), + RTE_CACHE_LINE_SIZE, + p->numa_node); + CHECK(r->regarray, ENOMEM); + + if (regarray->init_val) + for (i = 0; i < regarray->size; i++) + r->regarray[i] = regarray->init_val; + + r->size_mask = regarray->size - 1; + } + + return 0; +} + +static void +regarray_build_free(struct rte_swx_pipeline *p) +{ + uint32_t i; + + if (!p->regarray_runtime) + return; + + for (i = 0; i < p->n_regarrays; i++) { + struct regarray *regarray = regarray_find_by_id(p, i); + struct regarray_runtime *r = &p->regarray_runtime[i]; + + env_free(r->regarray, regarray->size * sizeof(uint64_t)); + } + + free(p->regarray_runtime); + p->regarray_runtime = NULL; +} + +static void +regarray_free(struct rte_swx_pipeline *p) +{ + regarray_build_free(p); + + for ( ; ; ) { + struct regarray *elem; + + elem = TAILQ_FIRST(&p->regarrays); + if (!elem) + break; + + TAILQ_REMOVE(&p->regarrays, elem, node); + free(elem); + } +} + /* * Pipeline. */ @@ -6851,6 +8076,7 @@ rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node) TAILQ_INIT(&pipeline->actions); TAILQ_INIT(&pipeline->table_types); TAILQ_INIT(&pipeline->tables); + TAILQ_INIT(&pipeline->regarrays); pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */ pipeline->numa_node = numa_node; @@ -6867,6 +8093,7 @@ rte_swx_pipeline_free(struct rte_swx_pipeline *p) free(p->instructions); + regarray_free(p); table_state_free(p); table_free(p); action_free(p); @@ -6951,10 +8178,15 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p) if (status) goto error; + status = regarray_build(p); + if (status) + goto error; + p->build_done = 1; return 0; error: + regarray_build_free(p); table_state_build_free(p); table_build_free(p); action_build_free(p); @@ -7015,6 +8247,7 @@ rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p, pipeline->n_ports_out = p->n_ports_out; pipeline->n_actions = n_actions; pipeline->n_tables = n_tables; + pipeline->n_regarrays = p->n_regarrays; return 0; } @@ -7222,3 +8455,64 @@ rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p, port->type->ops.stats_read(port->obj, stats); return 0; } + +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray) +{ + struct regarray *r; + + if (!p || !regarray) + return -EINVAL; + + r = regarray_find_by_id(p, regarray_id); + if (!r) + return -EINVAL; + + strcpy(regarray->name, r->name); + regarray->size = r->size; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name || !value) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + *value = r->regarray[regarray_index]; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + r->regarray[regarray_index] = value; + return 0; +} diff --git a/lib/librte_pipeline/rte_swx_pipeline.h b/lib/librte_pipeline/rte_swx_pipeline.h index f0a2cef77..1c9b2eb12 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.h +++ b/lib/librte_pipeline/rte_swx_pipeline.h @@ -616,6 +616,30 @@ rte_swx_pipeline_table_config(struct rte_swx_pipeline *p, const char *args, uint32_t size); +/** + * Pipeline register array configure + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Register array name. + * @param[in] size + * Number of registers in the array. Each register is 64-bit in size. + * @param[in] init_val + * Initial value for every register in the array. The recommended value is 0. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Register array with this name already exists. + */ +__rte_experimental +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size, + uint64_t init_val); + /** * Pipeline instructions configure * diff --git a/lib/librte_pipeline/rte_swx_pipeline_spec.c b/lib/librte_pipeline/rte_swx_pipeline_spec.c index f7884491b..0d0bca2ed 100644 --- a/lib/librte_pipeline/rte_swx_pipeline_spec.c +++ b/lib/librte_pipeline/rte_swx_pipeline_spec.c @@ -940,6 +940,81 @@ table_block_parse(struct table_spec *s, return -EINVAL; } +/* + * regarray. + * + * regarray NAME size SIZE initval INITVAL + */ +struct regarray_spec { + char *name; + uint64_t init_val; + uint32_t size; +}; + +static void +regarray_spec_free(struct regarray_spec *s) +{ + if (!s) + return; + + free(s->name); + s->name = NULL; +} + +static int +regarray_statement_parse(struct regarray_spec *s, + char **tokens, + uint32_t n_tokens, + uint32_t n_lines, + uint32_t *err_line, + const char **err_msg) +{ + char *p; + + /* Check format. */ + if ((n_tokens != 6) || + strcmp(tokens[2], "size") || + strcmp(tokens[4], "initval")) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid regarray statement."; + return -EINVAL; + } + + /* spec. */ + s->name = strdup(tokens[1]); + if (!s->name) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Memory allocation failed."; + return -ENOMEM; + } + + p = tokens[3]; + s->size = strtoul(p, &p, 0); + if (p[0] || !s->size) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid size argument."; + return -EINVAL; + } + + p = tokens[5]; + s->init_val = strtoull(p, &p, 0); + if (p[0]) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid initval argument."; + return -EINVAL; + } + + return 0; +} + /* * apply. * @@ -1066,6 +1141,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, struct metadata_spec metadata_spec = {0}; struct action_spec action_spec = {0}; struct table_spec table_spec = {0}; + struct regarray_spec regarray_spec = {0}; struct apply_spec apply_spec = {0}; uint32_t n_lines; uint32_t block_mask = 0; @@ -1405,6 +1481,34 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, continue; } + /* regarray. */ + if (!strcmp(tokens[0], "regarray")) { + status = regarray_statement_parse(®array_spec, + tokens, + n_tokens, + n_lines, + err_line, + err_msg); + if (status) + goto error; + + status = rte_swx_pipeline_regarray_config(p, + regarray_spec.name, + regarray_spec.size, + regarray_spec.init_val); + if (status) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Register array configuration error."; + goto error; + } + + regarray_spec_free(®array_spec); + + continue; + } + /* apply. */ if (!strcmp(tokens[0], "apply")) { status = apply_statement_parse(&block_mask, @@ -1457,6 +1561,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, metadata_spec_free(&metadata_spec); action_spec_free(&action_spec); table_spec_free(&table_spec); + regarray_spec_free(®array_spec); apply_spec_free(&apply_spec); return status; } diff --git a/lib/librte_pipeline/version.map b/lib/librte_pipeline/version.map index 87c826f1b..26a9edf47 100644 --- a/lib/librte_pipeline/version.map +++ b/lib/librte_pipeline/version.map @@ -102,4 +102,10 @@ EXPERIMENTAL { rte_swx_pipeline_table_state_get; rte_swx_pipeline_table_state_set; rte_swx_pipeline_table_type_register; + + #added in 21.05 + rte_swx_ctl_pipeline_regarray_read; + rte_swx_ctl_pipeline_regarray_write; + rte_swx_ctl_regarray_info_get; + rte_swx_pipeline_regarray_config; }; -- 2.17.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* [dpdk-dev] [PATCH v3 2/2] pipeline: add meter support to the SWX pipeline 2021-03-15 22:22 ` [dpdk-dev] [PATCH v3 1/2] " Cristian Dumitrescu @ 2021-03-15 22:22 ` Cristian Dumitrescu 2021-03-15 22:48 ` [dpdk-dev] [PATCH v4 1/2] pipeline: add register array support to " Cristian Dumitrescu 1 sibling, 0 replies; 8+ messages in thread From: Cristian Dumitrescu @ 2021-03-15 22:22 UTC (permalink / raw) To: dev Meter arrays are stateful objects that are updated by the data plane and configured & monitored by the control plane. The meters implement the RFC 2698 Two Rate Three Color Marker (trTCM) algorithm. Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com> --- examples/pipeline/cli.c | 471 ++++++ examples/pipeline/examples/meter.cli | 31 + examples/pipeline/examples/meter.spec | 63 + lib/librte_pipeline/rte_swx_ctl.h | 154 ++ lib/librte_pipeline/rte_swx_pipeline.c | 1684 ++++++++++++++++--- lib/librte_pipeline/rte_swx_pipeline.h | 22 + lib/librte_pipeline/rte_swx_pipeline_spec.c | 91 + lib/librte_pipeline/version.map | 7 + 8 files changed, 2327 insertions(+), 196 deletions(-) create mode 100644 examples/pipeline/examples/meter.cli create mode 100644 examples/pipeline/examples/meter.spec diff --git a/examples/pipeline/cli.c b/examples/pipeline/cli.c index cdf8f13f3..045525307 100644 --- a/examples/pipeline/cli.c +++ b/examples/pipeline/cli.c @@ -1108,6 +1108,398 @@ cmd_pipeline_regwr(char **tokens, } } +static const char cmd_pipeline_meter_profile_add_help[] = +"pipeline <pipeline_name> meter profile <profile_name> add " + "cir <cir> pir <pir> cbs <cbs> pbs <pbs>\n"; + +static void +cmd_pipeline_meter_profile_add(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct rte_meter_trtcm_params params; + struct pipeline *p; + const char *profile_name; + int status; + + if (n_tokens != 14) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + if (strcmp(tokens[3], "profile")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile"); + return; + } + + profile_name = tokens[4]; + + if (strcmp(tokens[5], "add")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "add"); + return; + } + + if (strcmp(tokens[6], "cir")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cir"); + return; + } + + if (parser_read_uint64(¶ms.cir, tokens[7])) { + snprintf(out, out_size, MSG_ARG_INVALID, "cir"); + return; + } + + if (strcmp(tokens[8], "pir")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pir"); + return; + } + + if (parser_read_uint64(¶ms.pir, tokens[9])) { + snprintf(out, out_size, MSG_ARG_INVALID, "pir"); + return; + } + + if (strcmp(tokens[10], "cbs")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cbs"); + return; + } + + if (parser_read_uint64(¶ms.cbs, tokens[11])) { + snprintf(out, out_size, MSG_ARG_INVALID, "cbs"); + return; + } + + if (strcmp(tokens[12], "pbs")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pbs"); + return; + } + + if (parser_read_uint64(¶ms.pbs, tokens[13])) { + snprintf(out, out_size, MSG_ARG_INVALID, "pbs"); + return; + } + + status = rte_swx_ctl_meter_profile_add(p->p, profile_name, ¶ms); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + +static const char cmd_pipeline_meter_profile_delete_help[] = +"pipeline <pipeline_name> meter profile <profile_name> delete\n"; + +static void +cmd_pipeline_meter_profile_delete(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *profile_name; + int status; + + if (n_tokens != 6) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + if (strcmp(tokens[3], "profile")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile"); + return; + } + + profile_name = tokens[4]; + + if (strcmp(tokens[5], "delete")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "delete"); + return; + } + + status = rte_swx_ctl_meter_profile_delete(p->p, profile_name); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + +static const char cmd_pipeline_meter_reset_help[] = +"pipeline <pipeline_name> meter <meter_array_name> from <index0> to <index1> " + "reset\n"; + +static void +cmd_pipeline_meter_reset(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint32_t idx0, idx1; + + if (n_tokens != 9) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + name = tokens[3]; + + if (strcmp(tokens[4], "from")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "from"); + return; + } + + if (parser_read_uint32(&idx0, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index0"); + return; + } + + if (strcmp(tokens[6], "to")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "to"); + return; + } + + if (parser_read_uint32(&idx1, tokens[7]) || (idx1 < idx0)) { + snprintf(out, out_size, MSG_ARG_INVALID, "index1"); + return; + } + + if (strcmp(tokens[8], "reset")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "reset"); + return; + } + + for ( ; idx0 <= idx1; idx0++) { + int status; + + status = rte_swx_ctl_meter_reset(p->p, name, idx0); + if (status) { + snprintf(out, out_size, "Command failed for index %u.\n", idx0); + return; + } + } +} + +static const char cmd_pipeline_meter_set_help[] = +"pipeline <pipeline_name> meter <meter_array_name> from <index0> to <index1> " + "set profile <profile_name>\n"; + +static void +cmd_pipeline_meter_set(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name, *profile_name; + uint32_t idx0, idx1; + + if (n_tokens != 11) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + name = tokens[3]; + + if (strcmp(tokens[4], "from")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "from"); + return; + } + + if (parser_read_uint32(&idx0, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index0"); + return; + } + + if (strcmp(tokens[6], "to")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "to"); + return; + } + + if (parser_read_uint32(&idx1, tokens[7]) || (idx1 < idx0)) { + snprintf(out, out_size, MSG_ARG_INVALID, "index1"); + return; + } + + if (strcmp(tokens[8], "set")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "set"); + return; + } + + if (strcmp(tokens[9], "profile")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile"); + return; + } + + profile_name = tokens[10]; + + for ( ; idx0 <= idx1; idx0++) { + int status; + + status = rte_swx_ctl_meter_set(p->p, name, idx0, profile_name); + if (status) { + snprintf(out, out_size, "Command failed for index %u.\n", idx0); + return; + } + } +} + +static const char cmd_pipeline_meter_stats_help[] = +"pipeline <pipeline_name> meter <meter_array_name> from <index0> to <index1> " + "stats\n"; + +static void +cmd_pipeline_meter_stats(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct rte_swx_ctl_meter_stats stats; + struct pipeline *p; + const char *name; + uint32_t idx0, idx1; + + if (n_tokens != 9) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + name = tokens[3]; + + if (strcmp(tokens[4], "from")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "from"); + return; + } + + if (parser_read_uint32(&idx0, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index0"); + return; + } + + if (strcmp(tokens[6], "to")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "to"); + return; + } + + if (parser_read_uint32(&idx1, tokens[7]) || (idx1 < idx0)) { + snprintf(out, out_size, MSG_ARG_INVALID, "index1"); + return; + } + + if (strcmp(tokens[8], "stats")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "stats"); + return; + } + + /* Table header. */ + snprintf(out, out_size, "+-%7s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+\n", + "-------", + "----------------", "----------------", "----------------", + "----------------", "----------------", "----------------"); + out_size -= strlen(out); + out += strlen(out); + + snprintf(out, out_size, "| %4s | %16s | %16s | %16s | %16s | %16s | %16s |\n", + "METER #", + "GREEN (packets)", "YELLOW (packets)", "RED (packets)", + "GREEN (bytes)", "YELLOW (bytes)", "RED (bytes)"); + out_size -= strlen(out); + out += strlen(out); + + snprintf(out, out_size, "+-%7s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+\n", + "-------", + "----------------", "----------------", "----------------", + "----------------", "----------------", "----------------"); + out_size -= strlen(out); + out += strlen(out); + + /* Table rows. */ + for ( ; idx0 <= idx1; idx0++) { + int status; + + status = rte_swx_ctl_meter_stats_read(p->p, name, idx0, &stats); + if (status) { + snprintf(out, out_size, "Pipeline meter stats error at index %u.\n", idx0); + out_size -= strlen(out); + out += strlen(out); + return; + } + + snprintf(out, out_size, "| %7d | %16" PRIx64 " | %16" PRIx64 " | %16" PRIx64 + " | %16" PRIx64 " | %16" PRIx64 " | %16" PRIx64 " |\n", + idx0, + stats.n_pkts[RTE_COLOR_GREEN], + stats.n_pkts[RTE_COLOR_YELLOW], + stats.n_pkts[RTE_COLOR_RED], + stats.n_bytes[RTE_COLOR_GREEN], + stats.n_bytes[RTE_COLOR_YELLOW], + stats.n_bytes[RTE_COLOR_RED]); + out_size -= strlen(out); + out += strlen(out); + } +} + static const char cmd_pipeline_stats_help[] = "pipeline <pipeline_name> stats\n"; @@ -1303,6 +1695,11 @@ cmd_help(char **tokens, "\tpipeline table update\n" "\tpipeline regrd\n" "\tpipeline regwr\n" + "\tpipeline meter profile add\n" + "\tpipeline meter profile delete\n" + "\tpipeline meter reset\n" + "\tpipeline meter set\n" + "\tpipeline meter stats\n" "\tpipeline stats\n" "\tthread pipeline enable\n" "\tthread pipeline disable\n\n"); @@ -1373,6 +1770,43 @@ cmd_help(char **tokens, return; } + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 4) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[1], "profile") + && !strcmp(tokens[1], "add")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_profile_add_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 4) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[1], "profile") + && !strcmp(tokens[1], "delete")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_profile_delete_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 3) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "reset")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_reset_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 3) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "set")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_set_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 3) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "stats")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_stats_help); + return; + } + if ((n_tokens == 3) && (strcmp(tokens[0], "thread") == 0) && (strcmp(tokens[1], "pipeline") == 0)) { @@ -1481,6 +1915,43 @@ cli_process(char *in, char *out, size_t out_size, void *obj) return; } + if ((n_tokens >= 6) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[3], "profile") == 0) && + (strcmp(tokens[5], "add") == 0)) { + cmd_pipeline_meter_profile_add(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 6) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[3], "profile") == 0) && + (strcmp(tokens[5], "delete") == 0)) { + cmd_pipeline_meter_profile_delete(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 9) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[8], "reset") == 0)) { + cmd_pipeline_meter_reset(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 9) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[8], "set") == 0)) { + cmd_pipeline_meter_set(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 9) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[8], "stats") == 0)) { + cmd_pipeline_meter_stats(tokens, n_tokens, out, out_size, obj); + return; + } + if ((n_tokens >= 3) && (strcmp(tokens[2], "stats") == 0)) { cmd_pipeline_stats(tokens, n_tokens, out, out_size, diff --git a/examples/pipeline/examples/meter.cli b/examples/pipeline/examples/meter.cli new file mode 100644 index 000000000..b29ed2402 --- /dev/null +++ b/examples/pipeline/examples/meter.cli @@ -0,0 +1,31 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2020 Intel Corporation + +; Example command line: +; ./build/examples/dpdk-pipeline -l0-1 -- -s ./examples/pipeline/examples/meter.cli + +mempool MEMPOOL0 buffer 2304 pool 32K cache 256 cpu 0 + +link LINK0 dev 0000:18:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK1 dev 0000:18:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK2 dev 0000:3b:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK3 dev 0000:3b:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on + +pipeline PIPELINE0 create 0 + +pipeline PIPELINE0 port in 0 link LINK0 rxq 0 bsz 32 +pipeline PIPELINE0 port in 1 link LINK1 rxq 0 bsz 32 +pipeline PIPELINE0 port in 2 link LINK2 rxq 0 bsz 32 +pipeline PIPELINE0 port in 3 link LINK3 rxq 0 bsz 32 + +pipeline PIPELINE0 port out 0 link LINK0 txq 0 bsz 32 +pipeline PIPELINE0 port out 1 link LINK1 txq 0 bsz 32 +pipeline PIPELINE0 port out 2 link LINK2 txq 0 bsz 32 +pipeline PIPELINE0 port out 3 link LINK3 txq 0 bsz 32 + +pipeline PIPELINE0 build ./examples/pipeline/examples/meter.spec + +pipeline PIPELINE0 meter profile platinum add cir 46000000 pir 138000000 cbs 1000000 pbs 1000000 +pipeline PIPELINE0 meter meters from 0 to 15 set profile platinum + +thread 1 pipeline PIPELINE0 enable diff --git a/examples/pipeline/examples/meter.spec b/examples/pipeline/examples/meter.spec new file mode 100644 index 000000000..43b8170d7 --- /dev/null +++ b/examples/pipeline/examples/meter.spec @@ -0,0 +1,63 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2021 Intel Corporation + +; This program is setting up an array of Two Rate Three Color Marker (trTCM) meters called "meters". +; Every input packet (Ethernet/IPv4) is metered by the meter at the location indexed by the IPv4 +; header "Source Address" field. All green packets are sent out on port 0, the yellow ones on port 1 +; and the red ones on port 2. +; +; The "meter stats" CLI command can be used to read the packet and byte statistics counters of any +; meter in the array. + +// +// Headers. +// +struct ethernet_h { + bit<48> dst_addr + bit<48> src_addr + bit<16> ethertype +} + +struct ipv4_h { + bit<8> ver_ihl + bit<8> diffserv + bit<16> total_len + bit<16> identification + bit<16> flags_offset + bit<8> ttl + bit<8> protocol + bit<16> hdr_checksum + bit<32> src_addr + bit<32> dst_addr +} + +header ethernet instanceof ethernet_h +header ipv4 instanceof ipv4_h + +// +// Meta-data. +// +struct metadata_t { + bit<32> port_in + bit<32> port_out +} + +metadata instanceof metadata_t + +// +// Registers. +// +metarray meters size 65536 + +// +// Pipeline. +// +apply { + rx m.port_in + extract h.ethernet + extract h.ipv4 + meter meters h.ipv4.src_addr h.ipv4.total_len 0 m.port_out + emit h.ethernet + emit h.ipv4 + tx m.port_out +} diff --git a/lib/librte_pipeline/rte_swx_ctl.h b/lib/librte_pipeline/rte_swx_ctl.h index c0a55c6f0..f191ccaf6 100644 --- a/lib/librte_pipeline/rte_swx_ctl.h +++ b/lib/librte_pipeline/rte_swx_ctl.h @@ -18,6 +18,7 @@ extern "C" { #include <stdio.h> #include <rte_compat.h> +#include <rte_meter.h> #include "rte_swx_port.h" #include "rte_swx_table.h" @@ -49,6 +50,9 @@ struct rte_swx_ctl_pipeline_info { /** Number of register arrays. */ uint32_t n_regarrays; + + /** Number of meter arrays. */ + uint32_t n_metarrays; }; /** @@ -636,6 +640,156 @@ rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, uint32_t regarray_index, uint64_t value); +/* + * Meter Array Query and Configuration API. + */ + +/** Meter array info. */ +struct rte_swx_ctl_metarray_info { + /** Meter array name. */ + char name[RTE_SWX_CTL_NAME_SIZE]; + + /** Meter array size. */ + uint32_t size; +}; + +/** + * Meter array info get + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_id + * Meter array ID (0 .. *n_metarrays* - 1). + * @param[out] metarray + * Meter array info. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p, + uint32_t metarray_id, + struct rte_swx_ctl_metarray_info *metarray); + +/** + * Meter profile add + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Meter profile name. + * @param[in] params + * Meter profile parameters. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Meter profile with this name already exists. + */ +__rte_experimental +int +rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p, + const char *name, + struct rte_meter_trtcm_params *params); + +/** + * Meter profile delete + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Meter profile name. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -EBUSY: Meter profile is currently in use. + */ +__rte_experimental +int +rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p, + const char *name); + +/** + * Meter reset + * + * Reset a meter within a given meter array to use the default profile. It is + * the resposibility of the control plane to make sure this meter is not used by + * the data plane pipeline before calling this function. + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_name + * Meter array name. + * @param[in] metarray_index + * Meter index within the meter array. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index); + +/** + * Meter set + * + * Set a meter within a given meter array to use a specific profile. It is the + * resposibility of the control plane to make sure this meter is not used by the + * data plane pipeline before calling this function. + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_name + * Meter array name. + * @param[in] metarray_index + * Meter index within the meter array. + * @param[in] profile_name + * Existing meter profile name. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_meter_set(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + const char *profile_name); + +/** Meter statistics counters. */ +struct rte_swx_ctl_meter_stats { + /** Number of packets tagged by the meter for each color. */ + uint64_t n_pkts[RTE_COLORS]; + + /** Number of bytes tagged by the meter for each color. */ + uint64_t n_bytes[RTE_COLORS]; +}; + +/** + * Meter statistics counters read + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_name + * Meter array name. + * @param[in] metarray_index + * Meter index within the meter array. + * @param[out] stats + * Meter statistics counters. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + struct rte_swx_ctl_meter_stats *stats); + /** * Pipeline control free * diff --git a/lib/librte_pipeline/rte_swx_pipeline.c b/lib/librte_pipeline/rte_swx_pipeline.c index 6d4a407ce..80d1cf925 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.c +++ b/lib/librte_pipeline/rte_swx_pipeline.c @@ -12,6 +12,8 @@ #include <rte_common.h> #include <rte_prefetch.h> #include <rte_byteorder.h> +#include <rte_cycles.h> +#include <rte_meter.h> #include "rte_swx_pipeline.h" #include "rte_swx_ctl.h" @@ -458,6 +460,31 @@ enum instruction_type { INSTR_REGADD_RIM, /* index = I, src = MEFT */ INSTR_REGADD_RII, /* index = I, src = I */ + /* metprefetch METARRAY index + * prefetch METARRAY[index] + * index = HMEFTI + */ + INSTR_METPREFETCH_H, /* index = H */ + INSTR_METPREFETCH_M, /* index = MEFT */ + INSTR_METPREFETCH_I, /* index = I */ + + /* meter METARRAY index length color_in color_out + * color_out = meter(METARRAY[index], length, color_in) + * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF + */ + INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */ + INSTR_METER_HHI, /* index = H, length = H, color_in = I */ + INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */ + INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */ + INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */ + INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */ + INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */ + INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */ + INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */ + INSTR_METER_IHI, /* index = I, length = H, color_in = I */ + INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */ + INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */ + /* table TABLE */ INSTR_TABLE, @@ -607,6 +634,25 @@ struct instr_regarray { }; }; +struct instr_meter { + uint8_t metarray_id; + uint8_t pad[3]; + + union { + struct instr_operand idx; + uint32_t idx_val; + }; + + struct instr_operand length; + + union { + struct instr_operand color_in; + uint32_t color_in_val; + }; + + struct instr_operand color_out; +}; + struct instr_dma { struct { uint8_t header_id[8]; @@ -642,6 +688,7 @@ struct instruction { struct instr_hdr_validity valid; struct instr_dst_src mov; struct instr_regarray regarray; + struct instr_meter meter; struct instr_dma dma; struct instr_dst_src alu; struct instr_table table; @@ -739,6 +786,43 @@ struct regarray_runtime { uint32_t size_mask; }; +/* + * Meter array. + */ +struct meter_profile { + TAILQ_ENTRY(meter_profile) node; + char name[RTE_SWX_NAME_SIZE]; + struct rte_meter_trtcm_params params; + struct rte_meter_trtcm_profile profile; + uint32_t n_users; +}; + +TAILQ_HEAD(meter_profile_tailq, meter_profile); + +struct metarray { + TAILQ_ENTRY(metarray) node; + char name[RTE_SWX_NAME_SIZE]; + uint32_t size; + uint32_t id; +}; + +TAILQ_HEAD(metarray_tailq, metarray); + +struct meter { + struct rte_meter_trtcm m; + struct meter_profile *profile; + enum rte_color color_mask; + uint8_t pad[20]; + + uint64_t n_pkts[RTE_COLORS]; + uint64_t n_bytes[RTE_COLORS]; +}; + +struct metarray_runtime { + struct meter *metarray; + uint32_t size_mask; +}; + /* * Pipeline. */ @@ -1115,12 +1199,15 @@ struct rte_swx_pipeline { struct table_type_tailq table_types; struct table_tailq tables; struct regarray_tailq regarrays; + struct meter_profile_tailq meter_profiles; + struct metarray_tailq metarrays; struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; struct rte_swx_table_state *table_state; struct regarray_runtime *regarray_runtime; + struct metarray_runtime *metarray_runtime; struct instruction *instructions; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; @@ -1132,6 +1219,7 @@ struct rte_swx_pipeline { uint32_t n_actions; uint32_t n_tables; uint32_t n_regarrays; + uint32_t n_metarrays; uint32_t n_headers; uint32_t thread_id; uint32_t port_id; @@ -5658,255 +5746,1078 @@ instr_regadd_rii_exec(struct rte_swx_pipeline *p) } /* - * jmp. + * metarray. */ -static struct action * -action_find(struct rte_swx_pipeline *p, const char *name); - -static int -instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(n_tokens == 2, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - instr->type = INSTR_JMP; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} - -static int -instr_jmp_valid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct header *h; - - CHECK(n_tokens == 3, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); - - instr->type = INSTR_JMP_VALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; - return 0; -} +static struct metarray * +metarray_find(struct rte_swx_pipeline *p, const char *name); static int -instr_jmp_invalid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, +instr_metprefetch_translate(struct rte_swx_pipeline *p, + struct action *action, char **tokens, int n_tokens, struct instruction *instr, - struct instruction_data *data) + struct instruction_data *data __rte_unused) { - struct header *h; + char *metarray = tokens[1], *idx = tokens[2]; + struct metarray *m; + struct field *fidx; + uint32_t idx_struct_id, idx_val; CHECK(n_tokens == 3, EINVAL); - strcpy(data->jmp_label, tokens[1]); + m = metarray_find(p, metarray); + CHECK(m, EINVAL); - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + /* METPREFETCH_H, METPREFETCH_M. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_METPREFETCH_M; + if (idx[0] == 'h') + instr->type = INSTR_METPREFETCH_H; - instr->type = INSTR_JMP_INVALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; + instr->meter.metarray_id = m->id; + instr->meter.idx.struct_id = (uint8_t)idx_struct_id; + instr->meter.idx.n_bits = fidx->n_bits; + instr->meter.idx.offset = fidx->offset / 8; + return 0; + } + + /* METPREFETCH_I. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_METPREFETCH_I; + instr->meter.metarray_id = m->id; + instr->meter.idx_val = idx_val; return 0; } static int -instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_meter_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3]; + char *color_in = tokens[4], *color_out = tokens[5]; + struct metarray *m; + struct field *fidx, *flength, *fcin, *fcout; + uint32_t idx_struct_id, length_struct_id; + uint32_t color_in_struct_id, color_out_struct_id; - strcpy(data->jmp_label, tokens[1]); + CHECK(n_tokens == 6, EINVAL); - instr->type = INSTR_JMP_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + m = metarray_find(p, metarray); + CHECK(m, EINVAL); -static int -instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + fidx = struct_field_parse(p, action, idx, &idx_struct_id); - strcpy(data->jmp_label, tokens[1]); + flength = struct_field_parse(p, action, length, &length_struct_id); + CHECK(flength, EINVAL); - instr->type = INSTR_JMP_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + fcin = struct_field_parse(p, action, color_in, &color_in_struct_id); -static int -instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id); + CHECK(fcout, EINVAL); - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */ + if (fidx && fcin) { + instr->type = INSTR_METER_MMM; + if (idx[0] == 'h' && length[0] == 'h') + instr->type = INSTR_METER_HHM; + if (idx[0] == 'h' && length[0] != 'h') + instr->type = INSTR_METER_HMM; + if (idx[0] != 'h' && length[0] == 'h') + instr->type = INSTR_METER_MHM; - strcpy(data->jmp_label, tokens[1]); + instr->meter.metarray_id = m->id; - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + instr->meter.idx.struct_id = (uint8_t)idx_struct_id; + instr->meter.idx.n_bits = fidx->n_bits; + instr->meter.idx.offset = fidx->offset / 8; - instr->type = INSTR_JMP_ACTION_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; -static int -instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id; + instr->meter.color_in.n_bits = fcin->n_bits; + instr->meter.color_in.offset = fcin->offset / 8; - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; - strcpy(data->jmp_label, tokens[1]); + return 0; + } - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */ + if (fidx && !fcin) { + uint32_t color_in_val = strtoul(color_in, &color_in, 0); + CHECK(!color_in[0], EINVAL); - instr->type = INSTR_JMP_ACTION_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + instr->type = INSTR_METER_MMI; + if (idx[0] == 'h' && length[0] == 'h') + instr->type = INSTR_METER_HHI; + if (idx[0] == 'h' && length[0] != 'h') + instr->type = INSTR_METER_HMI; + if (idx[0] != 'h' && length[0] == 'h') + instr->type = INSTR_METER_MHI; -static int -instr_jmp_eq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + instr->meter.metarray_id = m->id; - CHECK(n_tokens == 4, EINVAL); + instr->meter.idx.struct_id = (uint8_t)idx_struct_id; + instr->meter.idx.n_bits = fidx->n_bits; + instr->meter.idx.offset = fidx->offset / 8; - strcpy(data->jmp_label, tokens[1]); + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + instr->meter.color_in_val = color_in_val; - /* JMP_EQ or JMP_EQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_EQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_EQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; return 0; } - /* JMP_EQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */ + if (!fidx && fcin) { + uint32_t idx_val; - if (a[0] == 'h') - b_val = hton64(b_val) >> (64 - fa->n_bits); + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - instr->type = INSTR_JMP_EQ_I; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b_val = b_val; - return 0; -} + instr->type = INSTR_METER_IMM; + if (length[0] == 'h') + instr->type = INSTR_METER_IHM; -static int -instr_jmp_neq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + instr->meter.metarray_id = m->id; - CHECK(n_tokens == 4, EINVAL); + instr->meter.idx_val = idx_val; - strcpy(data->jmp_label, tokens[1]); + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id; + instr->meter.color_in.n_bits = fcin->n_bits; + instr->meter.color_in.offset = fcin->offset / 8; - /* JMP_NEQ or JMP_NEQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_NEQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_NEQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; return 0; } - /* JMP_NEQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* index = I, length = HMEFT, color_in = I, color_out = MEF. */ + if (!fidx && !fcin) { + uint32_t idx_val, color_in_val; - if (a[0] == 'h') + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + color_in_val = strtoul(color_in, &color_in, 0); + CHECK(!color_in[0], EINVAL); + + instr->type = INSTR_METER_IMI; + if (length[0] == 'h') + instr->type = INSTR_METER_IHI; + + instr->meter.metarray_id = m->id; + + instr->meter.idx_val = idx_val; + + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; + + instr->meter.color_in_val = color_in_val; + + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; + + return 0; + } + + CHECK (0, EINVAL); +} + +static inline struct meter * +instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return &r->metarray[idx]; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline struct meter * +instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; + + return &r->metarray[idx]; +} + +#else + +#define instr_meter_idx_nbo instr_meter_idx_hbo + +#endif + +static inline struct meter * +instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint64_t idx = ip->meter.idx_val & r->size_mask; + + return &r->metarray[idx]; +} + +static inline uint32_t +instr_meter_length_hbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); + uint64_t src = src64 & src64_mask; + + return (uint32_t)src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint32_t +instr_meter_length_nbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); + + return (uint32_t)src; +} + +#else + +#define instr_meter_length_nbo instr_meter_length_hbo + +#endif + +static inline enum rte_color +instr_meter_color_in_hbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); + uint64_t src = src64 & src64_mask; + + return (enum rte_color)src; +} + +static inline void +instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out) +{ + uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); + + uint64_t src = (uint64_t)color_out; + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +static inline void +instr_metprefetch_h_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + + TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_metprefetch_m_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + + TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_metprefetch_i_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + + TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_hhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_hhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_hmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} +static inline void +instr_meter_hmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_ihm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_ihi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_imm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} +static inline void +instr_meter_imi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +/* + * jmp. + */ +static struct action * +action_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_valid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_VALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_invalid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_INVALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_eq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_EQ or JMP_EQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_EQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_EQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_EQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = hton64(b_val) >> (64 - fa->n_bits); + + instr->type = INSTR_JMP_EQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = b_val; + return 0; +} + +static int +instr_jmp_neq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_NEQ or JMP_NEQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_NEQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_NEQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_NEQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') b_val = hton64(b_val) >> (64 - fa->n_bits); instr->type = INSTR_JMP_NEQ_I; @@ -6543,6 +7454,22 @@ instr_translate(struct rte_swx_pipeline *p, instr, data); + if (!strcmp(tokens[tpos], "metprefetch")) + return instr_metprefetch_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "meter")) + return instr_meter_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + if (!strcmp(tokens[tpos], "table")) return instr_table_translate(p, action, @@ -7210,6 +8137,23 @@ static instr_exec_t instruction_table[] = { [INSTR_REGADD_RIM] = instr_regadd_rim_exec, [INSTR_REGADD_RII] = instr_regadd_rii_exec, + [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec, + [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec, + [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec, + + [INSTR_METER_HHM] = instr_meter_hhm_exec, + [INSTR_METER_HHI] = instr_meter_hhi_exec, + [INSTR_METER_HMM] = instr_meter_hmm_exec, + [INSTR_METER_HMI] = instr_meter_hmi_exec, + [INSTR_METER_MHM] = instr_meter_mhm_exec, + [INSTR_METER_MHI] = instr_meter_mhi_exec, + [INSTR_METER_MMM] = instr_meter_mmm_exec, + [INSTR_METER_MMI] = instr_meter_mmi_exec, + [INSTR_METER_IHM] = instr_meter_ihm_exec, + [INSTR_METER_IHI] = instr_meter_ihi_exec, + [INSTR_METER_IMM] = instr_meter_imm_exec, + [INSTR_METER_IMI] = instr_meter_imi_exec, + [INSTR_TABLE] = instr_table_exec, [INSTR_EXTERN_OBJ] = instr_extern_obj_exec, [INSTR_EXTERN_FUNC] = instr_extern_func_exec, @@ -8048,6 +8992,182 @@ regarray_free(struct rte_swx_pipeline *p) } } +/* + * Meter array. + */ +static struct meter_profile * +meter_profile_find(struct rte_swx_pipeline *p, const char *name) +{ + struct meter_profile *elem; + + TAILQ_FOREACH(elem, &p->meter_profiles, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct metarray * +metarray_find(struct rte_swx_pipeline *p, const char *name) +{ + struct metarray *elem; + + TAILQ_FOREACH(elem, &p->metarrays, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct metarray * +metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id) +{ + struct metarray *elem = NULL; + + TAILQ_FOREACH(elem, &p->metarrays, node) + if (elem->id == id) + return elem; + + return NULL; +} + +int +rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size) +{ + struct metarray *m; + + CHECK(p, EINVAL); + + CHECK_NAME(name, EINVAL); + CHECK(!metarray_find(p, name), EEXIST); + + CHECK(size, EINVAL); + size = rte_align32pow2(size); + + /* Memory allocation. */ + m = calloc(1, sizeof(struct metarray)); + CHECK(m, ENOMEM); + + /* Node initialization. */ + strcpy(m->name, name); + m->size = size; + m->id = p->n_metarrays; + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->metarrays, m, node); + p->n_metarrays++; + + return 0; +} + +struct meter_profile meter_profile_default = { + .node = {0}, + .name = "", + .params = {0}, + + .profile = { + .cbs = 10000, + .pbs = 10000, + .cir_period = 1, + .cir_bytes_per_period = 1, + .pir_period = 1, + .pir_bytes_per_period = 1, + }, + + .n_users = 0, +}; + +static void +meter_init(struct meter *m) +{ + memset(m, 0, sizeof(struct meter)); + rte_meter_trtcm_config(&m->m, &meter_profile_default.profile); + m->profile = &meter_profile_default; + m->color_mask = RTE_COLOR_GREEN; + + meter_profile_default.n_users++; +} + +static int +metarray_build(struct rte_swx_pipeline *p) +{ + struct metarray *m; + + if (!p->n_metarrays) + return 0; + + p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime)); + CHECK(p->metarray_runtime, ENOMEM); + + TAILQ_FOREACH(m, &p->metarrays, node) { + struct metarray_runtime *r = &p->metarray_runtime[m->id]; + uint32_t i; + + r->metarray = env_malloc(m->size * sizeof(struct meter), + RTE_CACHE_LINE_SIZE, + p->numa_node); + CHECK(r->metarray, ENOMEM); + + for (i = 0; i < m->size; i++) + meter_init(&r->metarray[i]); + + r->size_mask = m->size - 1; + } + + return 0; +} + +static void +metarray_build_free(struct rte_swx_pipeline *p) +{ + uint32_t i; + + if (!p->metarray_runtime) + return; + + for (i = 0; i < p->n_metarrays; i++) { + struct metarray *m = metarray_find_by_id(p, i); + struct metarray_runtime *r = &p->metarray_runtime[i]; + + env_free(r->metarray, m->size * sizeof(struct meter)); + } + + free(p->metarray_runtime); + p->metarray_runtime = NULL; +} + +static void +metarray_free(struct rte_swx_pipeline *p) +{ + metarray_build_free(p); + + /* Meter arrays. */ + for ( ; ; ) { + struct metarray *elem; + + elem = TAILQ_FIRST(&p->metarrays); + if (!elem) + break; + + TAILQ_REMOVE(&p->metarrays, elem, node); + free(elem); + } + + /* Meter profiles. */ + for ( ; ; ) { + struct meter_profile *elem; + + elem = TAILQ_FIRST(&p->meter_profiles); + if (!elem) + break; + + TAILQ_REMOVE(&p->meter_profiles, elem, node); + free(elem); + } +} + /* * Pipeline. */ @@ -8077,6 +9197,8 @@ rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node) TAILQ_INIT(&pipeline->table_types); TAILQ_INIT(&pipeline->tables); TAILQ_INIT(&pipeline->regarrays); + TAILQ_INIT(&pipeline->meter_profiles); + TAILQ_INIT(&pipeline->metarrays); pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */ pipeline->numa_node = numa_node; @@ -8093,6 +9215,7 @@ rte_swx_pipeline_free(struct rte_swx_pipeline *p) free(p->instructions); + metarray_free(p); regarray_free(p); table_state_free(p); table_free(p); @@ -8182,10 +9305,15 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p) if (status) goto error; + status = metarray_build(p); + if (status) + goto error; + p->build_done = 1; return 0; error: + metarray_build_free(p); regarray_build_free(p); table_state_build_free(p); table_build_free(p); @@ -8248,6 +9376,7 @@ rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p, pipeline->n_actions = n_actions; pipeline->n_tables = n_tables; pipeline->n_regarrays = p->n_regarrays; + pipeline->n_metarrays = p->n_metarrays; return 0; } @@ -8516,3 +9645,166 @@ rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, r->regarray[regarray_index] = value; return 0; } + +int +rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p, + uint32_t metarray_id, + struct rte_swx_ctl_metarray_info *metarray) +{ + struct metarray *m; + + if (!p || !metarray) + return -EINVAL; + + m = metarray_find_by_id(p, metarray_id); + if (!m) + return -EINVAL; + + strcpy(metarray->name, m->name); + metarray->size = m->size; + return 0; +} + +int +rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p, + const char *name, + struct rte_meter_trtcm_params *params) +{ + struct meter_profile *mp; + int status; + + CHECK(p, EINVAL); + CHECK_NAME(name, EINVAL); + CHECK(params, EINVAL); + CHECK(!meter_profile_find(p, name), EEXIST); + + /* Node allocation. */ + mp = calloc(1, sizeof(struct meter_profile)); + CHECK(mp, ENOMEM); + + /* Node initialization. */ + strcpy(mp->name, name); + memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params)); + status = rte_meter_trtcm_profile_config(&mp->profile, params); + if (status) { + free(mp); + CHECK(0, EINVAL); + } + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node); + + return 0; +} + +int +rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p, + const char *name) +{ + struct meter_profile *mp; + + CHECK(p, EINVAL); + CHECK_NAME(name, EINVAL); + + mp = meter_profile_find(p, name); + CHECK(mp, EINVAL); + CHECK(!mp->n_users, EBUSY); + + /* Remove node from tailq. */ + TAILQ_REMOVE(&p->meter_profiles, mp, node); + free(mp); + + return 0; +} + +int +rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index) +{ + struct meter_profile *mp_old; + struct metarray *metarray; + struct metarray_runtime *metarray_runtime; + struct meter *m; + + CHECK(p, EINVAL); + CHECK_NAME(metarray_name, EINVAL); + + metarray = metarray_find(p, metarray_name); + CHECK(metarray, EINVAL); + CHECK(metarray_index < metarray->size, EINVAL); + + metarray_runtime = &p->metarray_runtime[metarray->id]; + m = &metarray_runtime->metarray[metarray_index]; + mp_old = m->profile; + + meter_init(m); + + mp_old->n_users--; + + return 0; +} + +int +rte_swx_ctl_meter_set(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + const char *profile_name) +{ + struct meter_profile *mp, *mp_old; + struct metarray *metarray; + struct metarray_runtime *metarray_runtime; + struct meter *m; + + CHECK(p, EINVAL); + CHECK_NAME(metarray_name, EINVAL); + + metarray = metarray_find(p, metarray_name); + CHECK(metarray, EINVAL); + CHECK(metarray_index < metarray->size, EINVAL); + + mp = meter_profile_find(p, profile_name); + CHECK(mp, EINVAL); + + metarray_runtime = &p->metarray_runtime[metarray->id]; + m = &metarray_runtime->metarray[metarray_index]; + mp_old = m->profile; + + memset(m, 0, sizeof(struct meter)); + rte_meter_trtcm_config(&m->m, &mp->profile); + m->profile = mp; + m->color_mask = RTE_COLORS; + + mp->n_users++; + mp_old->n_users--; + + return 0; +} + +int +rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + struct rte_swx_ctl_meter_stats *stats) +{ + struct metarray *metarray; + struct metarray_runtime *metarray_runtime; + struct meter *m; + + CHECK(p, EINVAL); + CHECK_NAME(metarray_name, EINVAL); + + metarray = metarray_find(p, metarray_name); + CHECK(metarray, EINVAL); + CHECK(metarray_index < metarray->size, EINVAL); + + CHECK(stats, EINVAL); + + metarray_runtime = &p->metarray_runtime[metarray->id]; + m = &metarray_runtime->metarray[metarray_index]; + + memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts)); + memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes)); + + return 0; +} \ No newline at end of file diff --git a/lib/librte_pipeline/rte_swx_pipeline.h b/lib/librte_pipeline/rte_swx_pipeline.h index 1c9b2eb12..156b1fd67 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.h +++ b/lib/librte_pipeline/rte_swx_pipeline.h @@ -640,6 +640,28 @@ rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, uint32_t size, uint64_t init_val); +/** + * Pipeline meter array configure + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Meter array name. + * @param[in] size + * Number of meters in the array. Each meter in the array implements the Two + * Rate Three Color Marker (trTCM) algorithm, as specified by RFC 2698. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Meter array with this name already exists. + */ +__rte_experimental +int +rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size); + /** * Pipeline instructions configure * diff --git a/lib/librte_pipeline/rte_swx_pipeline_spec.c b/lib/librte_pipeline/rte_swx_pipeline_spec.c index 0d0bca2ed..2e867d7bf 100644 --- a/lib/librte_pipeline/rte_swx_pipeline_spec.c +++ b/lib/librte_pipeline/rte_swx_pipeline_spec.c @@ -1015,6 +1015,68 @@ regarray_statement_parse(struct regarray_spec *s, return 0; } +/* + * metarray. + * + * metarray NAME size SIZE + */ +struct metarray_spec { + char *name; + uint32_t size; +}; + +static void +metarray_spec_free(struct metarray_spec *s) +{ + if (!s) + return; + + free(s->name); + s->name = NULL; +} + +static int +metarray_statement_parse(struct metarray_spec *s, + char **tokens, + uint32_t n_tokens, + uint32_t n_lines, + uint32_t *err_line, + const char **err_msg) +{ + char *p; + + /* Check format. */ + if ((n_tokens != 4) || strcmp(tokens[2], "size")) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid metarray statement."; + return -EINVAL; + } + + /* spec. */ + s->name = strdup(tokens[1]); + if (!s->name) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Memory allocation failed."; + return -ENOMEM; + } + + p = tokens[3]; + s->size = strtoul(p, &p, 0); + if (p[0] || !s->size) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid size argument."; + return -EINVAL; + } + + return 0; +} + /* * apply. * @@ -1142,6 +1204,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, struct action_spec action_spec = {0}; struct table_spec table_spec = {0}; struct regarray_spec regarray_spec = {0}; + struct metarray_spec metarray_spec = {0}; struct apply_spec apply_spec = {0}; uint32_t n_lines; uint32_t block_mask = 0; @@ -1509,6 +1572,33 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, continue; } + /* metarray. */ + if (!strcmp(tokens[0], "metarray")) { + status = metarray_statement_parse(&metarray_spec, + tokens, + n_tokens, + n_lines, + err_line, + err_msg); + if (status) + goto error; + + status = rte_swx_pipeline_metarray_config(p, + metarray_spec.name, + metarray_spec.size); + if (status) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Meter array configuration error."; + goto error; + } + + metarray_spec_free(&metarray_spec); + + continue; + } + /* apply. */ if (!strcmp(tokens[0], "apply")) { status = apply_statement_parse(&block_mask, @@ -1562,6 +1652,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, action_spec_free(&action_spec); table_spec_free(&table_spec); regarray_spec_free(®array_spec); + metarray_spec_free(&metarray_spec); apply_spec_free(&apply_spec); return status; } diff --git a/lib/librte_pipeline/version.map b/lib/librte_pipeline/version.map index 26a9edf47..2049d6d69 100644 --- a/lib/librte_pipeline/version.map +++ b/lib/librte_pipeline/version.map @@ -104,8 +104,15 @@ EXPERIMENTAL { rte_swx_pipeline_table_type_register; #added in 21.05 + rte_swx_ctl_metarray_info_get; + rte_swx_ctl_meter_profile_add; + rte_swx_ctl_meter_profile_delete; + rte_swx_ctl_meter_reset; + rte_swx_ctl_meter_set; + rte_swx_ctl_meter_stats_read; rte_swx_ctl_pipeline_regarray_read; rte_swx_ctl_pipeline_regarray_write; rte_swx_ctl_regarray_info_get; + rte_swx_pipeline_metarray_config; rte_swx_pipeline_regarray_config; }; -- 2.17.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* [dpdk-dev] [PATCH v4 1/2] pipeline: add register array support to SWX pipeline 2021-03-15 22:22 ` [dpdk-dev] [PATCH v3 1/2] " Cristian Dumitrescu 2021-03-15 22:22 ` [dpdk-dev] [PATCH v3 2/2] pipeline: add meter support to the " Cristian Dumitrescu @ 2021-03-15 22:48 ` Cristian Dumitrescu 2021-03-15 22:48 ` [dpdk-dev] [PATCH v4 2/2] pipeline: add meter support to the " Cristian Dumitrescu 2021-03-16 11:40 ` [dpdk-dev] [PATCH v5 1/2] pipeline: add register array support to " Cristian Dumitrescu 1 sibling, 2 replies; 8+ messages in thread From: Cristian Dumitrescu @ 2021-03-15 22:48 UTC (permalink / raw) To: dev Register arrays are stateful objects that can be read & modified by both the data plane and the control plane, as opposed to tables, which are read-only for data plane. One key use-case is the implementation of stats counters. Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com> --- examples/pipeline/cli.c | 125 ++ examples/pipeline/examples/registers.cli | 28 + examples/pipeline/examples/registers.spec | 67 + lib/librte_pipeline/rte_swx_ctl.h | 79 + lib/librte_pipeline/rte_swx_pipeline.c | 1602 +++++++++++++++++-- lib/librte_pipeline/rte_swx_pipeline.h | 24 + lib/librte_pipeline/rte_swx_pipeline_spec.c | 105 ++ lib/librte_pipeline/version.map | 6 + 8 files changed, 1882 insertions(+), 154 deletions(-) create mode 100644 examples/pipeline/examples/registers.cli create mode 100644 examples/pipeline/examples/registers.spec diff --git a/examples/pipeline/cli.c b/examples/pipeline/cli.c index 30c2dd34d..cdf8f13f3 100644 --- a/examples/pipeline/cli.c +++ b/examples/pipeline/cli.c @@ -1009,6 +1009,105 @@ cmd_pipeline_table_update(char **tokens, fclose(file_default); } +static const char cmd_pipeline_regrd_help[] = +"pipeline <pipeline_name> regrd <register_array_name> <index>\n"; + +static void +cmd_pipeline_regrd(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 5) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regrd")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regrd"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_read(p->p, name, idx, &value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } + + snprintf(out, out_size, "0x%" PRIx64 "\n", value); +} + +static const char cmd_pipeline_regwr_help[] = +"pipeline <pipeline_name> regwr <register_array_name> <index> <value>\n"; + +static void +cmd_pipeline_regwr(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 6) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regwr")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regwr"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + if (parser_read_uint64(&value, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "value"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_write(p->p, name, idx, value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + static const char cmd_pipeline_stats_help[] = "pipeline <pipeline_name> stats\n"; @@ -1202,6 +1301,8 @@ cmd_help(char **tokens, "\tpipeline port out\n" "\tpipeline build\n" "\tpipeline table update\n" + "\tpipeline regrd\n" + "\tpipeline regwr\n" "\tpipeline stats\n" "\tthread pipeline enable\n" "\tthread pipeline disable\n\n"); @@ -1254,6 +1355,18 @@ cmd_help(char **tokens, return; } + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regrd") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regrd_help); + return; + } + + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regwr") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regwr_help); + return; + } + if ((strcmp(tokens[0], "pipeline") == 0) && (n_tokens == 2) && (strcmp(tokens[1], "stats") == 0)) { snprintf(out, out_size, "\n%s\n", cmd_pipeline_stats_help); @@ -1356,6 +1469,18 @@ cli_process(char *in, char *out, size_t out_size, void *obj) return; } + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regrd") == 0)) { + cmd_pipeline_regrd(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regwr") == 0)) { + cmd_pipeline_regwr(tokens, n_tokens, out, out_size, obj); + return; + } + if ((n_tokens >= 3) && (strcmp(tokens[2], "stats") == 0)) { cmd_pipeline_stats(tokens, n_tokens, out, out_size, diff --git a/examples/pipeline/examples/registers.cli b/examples/pipeline/examples/registers.cli new file mode 100644 index 000000000..8d026294c --- /dev/null +++ b/examples/pipeline/examples/registers.cli @@ -0,0 +1,28 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2020 Intel Corporation + +; Example command line: +; ./build/examples/dpdk-pipeline -l0-1 -- -s ./examples/pipeline/examples/registers.cli + +mempool MEMPOOL0 buffer 2304 pool 32K cache 256 cpu 0 + +link LINK0 dev 0000:18:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK1 dev 0000:18:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK2 dev 0000:3b:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK3 dev 0000:3b:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on + +pipeline PIPELINE0 create 0 + +pipeline PIPELINE0 port in 0 link LINK0 rxq 0 bsz 32 +pipeline PIPELINE0 port in 1 link LINK1 rxq 0 bsz 32 +pipeline PIPELINE0 port in 2 link LINK2 rxq 0 bsz 32 +pipeline PIPELINE0 port in 3 link LINK3 rxq 0 bsz 32 + +pipeline PIPELINE0 port out 0 link LINK0 txq 0 bsz 32 +pipeline PIPELINE0 port out 1 link LINK1 txq 0 bsz 32 +pipeline PIPELINE0 port out 2 link LINK2 txq 0 bsz 32 +pipeline PIPELINE0 port out 3 link LINK3 txq 0 bsz 32 + +pipeline PIPELINE0 build ./examples/pipeline/examples/registers.spec + +thread 1 pipeline PIPELINE0 enable diff --git a/examples/pipeline/examples/registers.spec b/examples/pipeline/examples/registers.spec new file mode 100644 index 000000000..74a014ad0 --- /dev/null +++ b/examples/pipeline/examples/registers.spec @@ -0,0 +1,67 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2021 Intel Corporation + +; This program is setting up two register arrays called "pkt_counters" and "byte_counters". +; On every input packet (Ethernet/IPv4), the "pkt_counters" register at location indexed by +; the IPv4 header "Source Address" field is incremented, while the same location in the +; "byte_counters" array accummulates the value of the IPv4 header "Total Length" field. +; +; The "regrd" and "regwr" CLI commands can be used to read and write the current value of +; any register array location. + +// +// Headers. +// +struct ethernet_h { + bit<48> dst_addr + bit<48> src_addr + bit<16> ethertype +} + +struct ipv4_h { + bit<8> ver_ihl + bit<8> diffserv + bit<16> total_len + bit<16> identification + bit<16> flags_offset + bit<8> ttl + bit<8> protocol + bit<16> hdr_checksum + bit<32> src_addr + bit<32> dst_addr +} + +header ethernet instanceof ethernet_h +header ipv4 instanceof ipv4_h + +// +// Meta-data. +// +struct metadata_t { + bit<32> port_in + bit<32> port_out +} + +metadata instanceof metadata_t + +// +// Registers. +// +regarray pkt_counters size 65536 initval 0 +regarray byte_counters size 65536 initval 0 + +// +// Pipeline. +// +apply { + rx m.port_in + extract h.ethernet + extract h.ipv4 + regadd pkt_counters h.ipv4.src_addr 1 + regadd byte_counters h.ipv4.src_addr h.ipv4.total_len + mov m.port_out m.port_in + xor m.port_out 1 + emit h.ethernet + emit h.ipv4 + tx m.port_out +} diff --git a/lib/librte_pipeline/rte_swx_ctl.h b/lib/librte_pipeline/rte_swx_ctl.h index 530671db1..c0a55c6f0 100644 --- a/lib/librte_pipeline/rte_swx_ctl.h +++ b/lib/librte_pipeline/rte_swx_ctl.h @@ -46,6 +46,9 @@ struct rte_swx_ctl_pipeline_info { /** Number of tables. */ uint32_t n_tables; + + /** Number of register arrays. */ + uint32_t n_regarrays; }; /** @@ -557,6 +560,82 @@ rte_swx_ctl_pipeline_table_fprintf(FILE *f, struct rte_swx_ctl_pipeline *ctl, const char *table_name); +/* + * Register Array Query API. + */ + +/** Register array info. */ +struct rte_swx_ctl_regarray_info { + /** Register array name. */ + char name[RTE_SWX_CTL_NAME_SIZE]; + + /** Register array size. */ + uint32_t size; +}; + +/** + * Register array info get + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_id + * Register array ID (0 .. *n_regarrays* - 1). + * @param[out] regarray + * Register array info. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray); + +/** + * Register read + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[out] value + * Current register value. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value); + +/** + * Register write + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[in] value + * Value to be written to the register. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value); + /** * Pipeline control free * diff --git a/lib/librte_pipeline/rte_swx_pipeline.c b/lib/librte_pipeline/rte_swx_pipeline.c index eaaed7a0a..5be87ca4f 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.c +++ b/lib/librte_pipeline/rte_swx_pipeline.c @@ -45,9 +45,59 @@ do { \ #define TRACE(...) #endif +/* + * Environment. + */ #define ntoh64(x) rte_be_to_cpu_64(x) #define hton64(x) rte_cpu_to_be_64(x) +#ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE + +#include <rte_malloc.h> + +static void * +env_malloc(size_t size, size_t alignment, int numa_node) +{ + return rte_zmalloc_socket(NULL, size, alignment, numa_node); +} + +static void +env_free(void *start, size_t size __rte_unused) +{ + rte_free(start); +} + +#else + +#include <numa.h> + +static void * +env_malloc(size_t size, size_t alignment __rte_unused, int numa_node) +{ + void *start; + + if (numa_available() == -1) + return NULL; + + start = numa_alloc_onnode(size, numa_node); + if (!start) + return NULL; + + memset(start, 0, size); + return start; +} + +static void +env_free(void *start, size_t size) +{ + if (numa_available() == -1) + return; + + numa_free(start, size); +} + +#endif + /* * Struct. */ @@ -361,6 +411,53 @@ enum instruction_type { INSTR_ALU_SHR_MI, /* dst = MEF, src = I */ INSTR_ALU_SHR_HI, /* dst = H, src = I */ + /* regprefetch REGARRAY index + * prefetch REGARRAY[index] + * index = HMEFTI + */ + INSTR_REGPREFETCH_RH, /* index = H */ + INSTR_REGPREFETCH_RM, /* index = MEFT */ + INSTR_REGPREFETCH_RI, /* index = I */ + + /* regrd dst REGARRAY index + * dst = REGARRAY[index] + * dst = HMEF, index = HMEFTI + */ + INSTR_REGRD_HRH, /* dst = H, index = H */ + INSTR_REGRD_HRM, /* dst = H, index = MEFT */ + INSTR_REGRD_HRI, /* dst = H, index = I */ + INSTR_REGRD_MRH, /* dst = MEF, index = H */ + INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */ + INSTR_REGRD_MRI, /* dst = MEF, index = I */ + + /* regwr REGARRAY index src + * REGARRAY[index] = src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGWR_RHH, /* index = H, src = H */ + INSTR_REGWR_RHM, /* index = H, src = MEFT */ + INSTR_REGWR_RHI, /* index = H, src = I */ + INSTR_REGWR_RMH, /* index = MEFT, src = H */ + INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGWR_RMI, /* index = MEFT, src = I */ + INSTR_REGWR_RIH, /* index = I, src = H */ + INSTR_REGWR_RIM, /* index = I, src = MEFT */ + INSTR_REGWR_RII, /* index = I, src = I */ + + /* regadd REGARRAY index src + * REGARRAY[index] += src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGADD_RHH, /* index = H, src = H */ + INSTR_REGADD_RHM, /* index = H, src = MEFT */ + INSTR_REGADD_RHI, /* index = H, src = I */ + INSTR_REGADD_RMH, /* index = MEFT, src = H */ + INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGADD_RMI, /* index = MEFT, src = I */ + INSTR_REGADD_RIH, /* index = I, src = H */ + INSTR_REGADD_RIM, /* index = I, src = MEFT */ + INSTR_REGADD_RII, /* index = I, src = I */ + /* table TABLE */ INSTR_TABLE, @@ -495,6 +592,21 @@ struct instr_dst_src { }; }; +struct instr_regarray { + uint8_t regarray_id; + uint8_t pad[3]; + + union { + struct instr_operand idx; + uint32_t idx_val; + }; + + union { + struct instr_operand dstsrc; + uint64_t dstsrc_val; + }; +}; + struct instr_dma { struct { uint8_t header_id[8]; @@ -529,6 +641,7 @@ struct instruction { struct instr_io io; struct instr_hdr_validity valid; struct instr_dst_src mov; + struct instr_regarray regarray; struct instr_dma dma; struct instr_dst_src alu; struct instr_table table; @@ -608,6 +721,24 @@ struct table_runtime { uint8_t **key; }; +/* + * Register array. + */ +struct regarray { + TAILQ_ENTRY(regarray) node; + char name[RTE_SWX_NAME_SIZE]; + uint64_t init_val; + uint32_t size; + uint32_t id; +}; + +TAILQ_HEAD(regarray_tailq, regarray); + +struct regarray_runtime { + uint64_t *regarray; + uint32_t size_mask; +}; + /* * Pipeline. */ @@ -983,11 +1114,13 @@ struct rte_swx_pipeline { struct action_tailq actions; struct table_type_tailq table_types; struct table_tailq tables; + struct regarray_tailq regarrays; struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; struct rte_swx_table_state *table_state; + struct regarray_runtime *regarray_runtime; struct instruction *instructions; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; @@ -998,6 +1131,7 @@ struct rte_swx_pipeline { uint32_t n_extern_funcs; uint32_t n_actions; uint32_t n_tables; + uint32_t n_regarrays; uint32_t n_headers; uint32_t thread_id; uint32_t port_id; @@ -4622,222 +4756,1124 @@ instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p) } /* - * jmp. + * Register array. */ -static struct action * -action_find(struct rte_swx_pipeline *p, const char *name); - -static int -instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(n_tokens == 2, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - instr->type = INSTR_JMP; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name); static int -instr_jmp_valid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regprefetch_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *regarray = tokens[1], *idx = tokens[2]; + struct regarray *r; + struct field *fidx; + uint32_t idx_struct_id, idx_val; CHECK(n_tokens == 3, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); + + /* REGPREFETCH_RH, REGPREFETCH_RM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGPREFETCH_RM; + if (idx[0] == 'h') + instr->type = INSTR_REGPREFETCH_RH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = 0; /* Unused. */ + return 0; + } - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + /* REGPREFETCH_RI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - instr->type = INSTR_JMP_VALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; + instr->type = INSTR_REGPREFETCH_RI; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = 0; /* Unused. */ return 0; } static int -instr_jmp_invalid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regrd_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3]; + struct regarray *r; + struct field *fdst, *fidx; + uint32_t dst_struct_id, idx_struct_id, idx_val; - CHECK(n_tokens == 3, EINVAL); + CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); - instr->type = INSTR_JMP_INVALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; - return 0; -} + /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGRD_MRM; + if (dst[0] == 'h' && idx[0] != 'h') + instr->type = INSTR_REGRD_HRM; + if (dst[0] != 'h' && idx[0] == 'h') + instr->type = INSTR_REGRD_MRH; + if (dst[0] == 'h' && idx[0] == 'h') + instr->type = INSTR_REGRD_HRH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; + return 0; + } -static int -instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + /* REGRD_MRI, REGRD_HRI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGRD_MRI; + if (dst[0] == 'h') + instr->type = INSTR_REGRD_HRI; - instr->type = INSTR_JMP_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; return 0; } static int -instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regwr_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; - strcpy(data->jmp_label, tokens[1]); + CHECK(n_tokens == 4, EINVAL); - instr->type = INSTR_JMP_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + r = regarray_find(p, regarray); + CHECK(r, EINVAL); -static int -instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGWR_RMM; + if (idx[0] == 'h' && src[0] != 'h') + instr->type = INSTR_REGWR_RHM; + if (idx[0] != 'h' && src[0] == 'h') + instr->type = INSTR_REGWR_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGWR_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + /* REGWR_RHI, REGWR_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGWR_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGWR_RHI; - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; + return 0; + } - instr->type = INSTR_JMP_ACTION_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + /* REGWR_RIH, REGWR_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); -static int -instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + instr->type = INSTR_REGWR_RIM; + if (src[0] == 'h') + instr->type = INSTR_REGWR_RIH; - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - strcpy(data->jmp_label, tokens[1]); + /* REGWR_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGWR_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; - instr->type = INSTR_JMP_ACTION_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; return 0; } static int -instr_jmp_eq_translate(struct rte_swx_pipeline *p, +instr_regadd_translate(struct rte_swx_pipeline *p, struct action *action, char **tokens, int n_tokens, struct instruction *instr, - struct instruction_data *data) + struct instruction_data *data __rte_unused) { - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGADD_RMM; + if (idx[0] == 'h' && src[0] != 'h') + instr->type = INSTR_REGADD_RHM; + if (idx[0] != 'h' && src[0] == 'h') + instr->type = INSTR_REGADD_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGADD_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - /* JMP_EQ or JMP_EQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_EQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_EQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + /* REGADD_RHI, REGADD_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; + instr->type = INSTR_REGADD_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGADD_RHI; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; return 0; } - /* JMP_EQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* REGADD_RIH, REGADD_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - if (a[0] == 'h') - b_val = hton64(b_val) >> (64 - fa->n_bits); + instr->type = INSTR_REGADD_RIM; + if (src[0] == 'h') + instr->type = INSTR_REGADD_RIH; - instr->type = INSTR_JMP_EQ_I; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b_val = b_val; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } + + /* REGADD_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); + + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGADD_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; return 0; } -static int -instr_jmp_neq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +static inline uint64_t * +instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip) { - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + return r->regarray; +} + +static inline uint64_t +instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return idx; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; + + return idx; +} + +#else + +#define instr_regarray_idx_nbo instr_regarray_idx_hbo + +#endif + +static inline uint64_t +instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint64_t idx = ip->regarray.idx_val & r->size_mask; + + return idx; +} + +static inline uint64_t +instr_regarray_src_hbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + uint64_t src = src64 & src64_mask; + + return src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_src_nbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); + + return src; +} + +#else + +#define instr_regarray_src_nbo instr_regarray_src_hbo + +#endif + +static inline void +instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); + +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline void +instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +#else + +#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set + +#endif + +static inline void +instr_regprefetch_rh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + rte_prefetch0(®array[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_rm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + rte_prefetch0(®array[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_ri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + rte_prefetch0(®array[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +/* + * jmp. + */ +static struct action * +action_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_valid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_VALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_invalid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_INVALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_eq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_EQ or JMP_EQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_EQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_EQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_EQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = hton64(b_val) >> (64 - fa->n_bits); + + instr->type = INSTR_JMP_EQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = b_val; + return 0; +} + +static int +instr_jmp_neq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; uint64_t b_val; uint32_t a_struct_id, b_struct_id; @@ -5475,6 +6511,38 @@ instr_translate(struct rte_swx_pipeline *p, instr, data); + if (!strcmp(tokens[tpos], "regprefetch")) + return instr_regprefetch_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regrd")) + return instr_regrd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regwr")) + return instr_regwr_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regadd")) + return instr_regadd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + if (!strcmp(tokens[tpos], "table")) return instr_table_translate(p, action, @@ -6111,6 +7179,37 @@ static instr_exec_t instruction_table[] = { [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec, [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec, + [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec, + [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec, + [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec, + + [INSTR_REGRD_HRH] = instr_regrd_hrh_exec, + [INSTR_REGRD_HRM] = instr_regrd_hrm_exec, + [INSTR_REGRD_MRH] = instr_regrd_mrh_exec, + [INSTR_REGRD_MRM] = instr_regrd_mrm_exec, + [INSTR_REGRD_HRI] = instr_regrd_hri_exec, + [INSTR_REGRD_MRI] = instr_regrd_mri_exec, + + [INSTR_REGWR_RHH] = instr_regwr_rhh_exec, + [INSTR_REGWR_RHM] = instr_regwr_rhm_exec, + [INSTR_REGWR_RMH] = instr_regwr_rmh_exec, + [INSTR_REGWR_RMM] = instr_regwr_rmm_exec, + [INSTR_REGWR_RHI] = instr_regwr_rhi_exec, + [INSTR_REGWR_RMI] = instr_regwr_rmi_exec, + [INSTR_REGWR_RIH] = instr_regwr_rih_exec, + [INSTR_REGWR_RIM] = instr_regwr_rim_exec, + [INSTR_REGWR_RII] = instr_regwr_rii_exec, + + [INSTR_REGADD_RHH] = instr_regadd_rhh_exec, + [INSTR_REGADD_RHM] = instr_regadd_rhm_exec, + [INSTR_REGADD_RMH] = instr_regadd_rmh_exec, + [INSTR_REGADD_RMM] = instr_regadd_rmm_exec, + [INSTR_REGADD_RHI] = instr_regadd_rhi_exec, + [INSTR_REGADD_RMI] = instr_regadd_rmi_exec, + [INSTR_REGADD_RIH] = instr_regadd_rih_exec, + [INSTR_REGADD_RIM] = instr_regadd_rim_exec, + [INSTR_REGADD_RII] = instr_regadd_rii_exec, + [INSTR_TABLE] = instr_table_exec, [INSTR_EXTERN_OBJ] = instr_extern_obj_exec, [INSTR_EXTERN_FUNC] = instr_extern_func_exec, @@ -6823,6 +7922,132 @@ table_free(struct rte_swx_pipeline *p) } } +/* + * Register array. + */ +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name) +{ + struct regarray *elem; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct regarray * +regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id) +{ + struct regarray *elem = NULL; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (elem->id == id) + return elem; + + return NULL; +} + +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size, + uint64_t init_val) +{ + struct regarray *r; + + CHECK(p, EINVAL); + + CHECK_NAME(name, EINVAL); + CHECK(!regarray_find(p, name), EEXIST); + + CHECK(size, EINVAL); + size = rte_align32pow2(size); + + /* Memory allocation. */ + r = calloc(1, sizeof(struct regarray)); + CHECK(r, ENOMEM); + + /* Node initialization. */ + strcpy(r->name, name); + r->init_val = init_val; + r->size = size; + r->id = p->n_regarrays; + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->regarrays, r, node); + p->n_regarrays++; + + return 0; +} + +static int +regarray_build(struct rte_swx_pipeline *p) +{ + struct regarray *regarray; + + if (!p->n_regarrays) + return 0; + + p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime)); + CHECK(p->regarray_runtime, ENOMEM); + + TAILQ_FOREACH(regarray, &p->regarrays, node) { + struct regarray_runtime *r = &p->regarray_runtime[regarray->id]; + uint32_t i; + + r->regarray = env_malloc(regarray->size * sizeof(uint64_t), + RTE_CACHE_LINE_SIZE, + p->numa_node); + CHECK(r->regarray, ENOMEM); + + if (regarray->init_val) + for (i = 0; i < regarray->size; i++) + r->regarray[i] = regarray->init_val; + + r->size_mask = regarray->size - 1; + } + + return 0; +} + +static void +regarray_build_free(struct rte_swx_pipeline *p) +{ + uint32_t i; + + if (!p->regarray_runtime) + return; + + for (i = 0; i < p->n_regarrays; i++) { + struct regarray *regarray = regarray_find_by_id(p, i); + struct regarray_runtime *r = &p->regarray_runtime[i]; + + env_free(r->regarray, regarray->size * sizeof(uint64_t)); + } + + free(p->regarray_runtime); + p->regarray_runtime = NULL; +} + +static void +regarray_free(struct rte_swx_pipeline *p) +{ + regarray_build_free(p); + + for ( ; ; ) { + struct regarray *elem; + + elem = TAILQ_FIRST(&p->regarrays); + if (!elem) + break; + + TAILQ_REMOVE(&p->regarrays, elem, node); + free(elem); + } +} + /* * Pipeline. */ @@ -6851,6 +8076,7 @@ rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node) TAILQ_INIT(&pipeline->actions); TAILQ_INIT(&pipeline->table_types); TAILQ_INIT(&pipeline->tables); + TAILQ_INIT(&pipeline->regarrays); pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */ pipeline->numa_node = numa_node; @@ -6867,6 +8093,7 @@ rte_swx_pipeline_free(struct rte_swx_pipeline *p) free(p->instructions); + regarray_free(p); table_state_free(p); table_free(p); action_free(p); @@ -6951,10 +8178,15 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p) if (status) goto error; + status = regarray_build(p); + if (status) + goto error; + p->build_done = 1; return 0; error: + regarray_build_free(p); table_state_build_free(p); table_build_free(p); action_build_free(p); @@ -7015,6 +8247,7 @@ rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p, pipeline->n_ports_out = p->n_ports_out; pipeline->n_actions = n_actions; pipeline->n_tables = n_tables; + pipeline->n_regarrays = p->n_regarrays; return 0; } @@ -7222,3 +8455,64 @@ rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p, port->type->ops.stats_read(port->obj, stats); return 0; } + +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray) +{ + struct regarray *r; + + if (!p || !regarray) + return -EINVAL; + + r = regarray_find_by_id(p, regarray_id); + if (!r) + return -EINVAL; + + strcpy(regarray->name, r->name); + regarray->size = r->size; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name || !value) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + *value = r->regarray[regarray_index]; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + r->regarray[regarray_index] = value; + return 0; +} diff --git a/lib/librte_pipeline/rte_swx_pipeline.h b/lib/librte_pipeline/rte_swx_pipeline.h index f0a2cef77..1c9b2eb12 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.h +++ b/lib/librte_pipeline/rte_swx_pipeline.h @@ -616,6 +616,30 @@ rte_swx_pipeline_table_config(struct rte_swx_pipeline *p, const char *args, uint32_t size); +/** + * Pipeline register array configure + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Register array name. + * @param[in] size + * Number of registers in the array. Each register is 64-bit in size. + * @param[in] init_val + * Initial value for every register in the array. The recommended value is 0. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Register array with this name already exists. + */ +__rte_experimental +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size, + uint64_t init_val); + /** * Pipeline instructions configure * diff --git a/lib/librte_pipeline/rte_swx_pipeline_spec.c b/lib/librte_pipeline/rte_swx_pipeline_spec.c index f7884491b..0d0bca2ed 100644 --- a/lib/librte_pipeline/rte_swx_pipeline_spec.c +++ b/lib/librte_pipeline/rte_swx_pipeline_spec.c @@ -940,6 +940,81 @@ table_block_parse(struct table_spec *s, return -EINVAL; } +/* + * regarray. + * + * regarray NAME size SIZE initval INITVAL + */ +struct regarray_spec { + char *name; + uint64_t init_val; + uint32_t size; +}; + +static void +regarray_spec_free(struct regarray_spec *s) +{ + if (!s) + return; + + free(s->name); + s->name = NULL; +} + +static int +regarray_statement_parse(struct regarray_spec *s, + char **tokens, + uint32_t n_tokens, + uint32_t n_lines, + uint32_t *err_line, + const char **err_msg) +{ + char *p; + + /* Check format. */ + if ((n_tokens != 6) || + strcmp(tokens[2], "size") || + strcmp(tokens[4], "initval")) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid regarray statement."; + return -EINVAL; + } + + /* spec. */ + s->name = strdup(tokens[1]); + if (!s->name) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Memory allocation failed."; + return -ENOMEM; + } + + p = tokens[3]; + s->size = strtoul(p, &p, 0); + if (p[0] || !s->size) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid size argument."; + return -EINVAL; + } + + p = tokens[5]; + s->init_val = strtoull(p, &p, 0); + if (p[0]) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid initval argument."; + return -EINVAL; + } + + return 0; +} + /* * apply. * @@ -1066,6 +1141,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, struct metadata_spec metadata_spec = {0}; struct action_spec action_spec = {0}; struct table_spec table_spec = {0}; + struct regarray_spec regarray_spec = {0}; struct apply_spec apply_spec = {0}; uint32_t n_lines; uint32_t block_mask = 0; @@ -1405,6 +1481,34 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, continue; } + /* regarray. */ + if (!strcmp(tokens[0], "regarray")) { + status = regarray_statement_parse(®array_spec, + tokens, + n_tokens, + n_lines, + err_line, + err_msg); + if (status) + goto error; + + status = rte_swx_pipeline_regarray_config(p, + regarray_spec.name, + regarray_spec.size, + regarray_spec.init_val); + if (status) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Register array configuration error."; + goto error; + } + + regarray_spec_free(®array_spec); + + continue; + } + /* apply. */ if (!strcmp(tokens[0], "apply")) { status = apply_statement_parse(&block_mask, @@ -1457,6 +1561,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, metadata_spec_free(&metadata_spec); action_spec_free(&action_spec); table_spec_free(&table_spec); + regarray_spec_free(®array_spec); apply_spec_free(&apply_spec); return status; } diff --git a/lib/librte_pipeline/version.map b/lib/librte_pipeline/version.map index 87c826f1b..26a9edf47 100644 --- a/lib/librte_pipeline/version.map +++ b/lib/librte_pipeline/version.map @@ -102,4 +102,10 @@ EXPERIMENTAL { rte_swx_pipeline_table_state_get; rte_swx_pipeline_table_state_set; rte_swx_pipeline_table_type_register; + + #added in 21.05 + rte_swx_ctl_pipeline_regarray_read; + rte_swx_ctl_pipeline_regarray_write; + rte_swx_ctl_regarray_info_get; + rte_swx_pipeline_regarray_config; }; -- 2.17.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* [dpdk-dev] [PATCH v4 2/2] pipeline: add meter support to the SWX pipeline 2021-03-15 22:48 ` [dpdk-dev] [PATCH v4 1/2] pipeline: add register array support to " Cristian Dumitrescu @ 2021-03-15 22:48 ` Cristian Dumitrescu 2021-03-16 11:40 ` [dpdk-dev] [PATCH v5 1/2] pipeline: add register array support to " Cristian Dumitrescu 1 sibling, 0 replies; 8+ messages in thread From: Cristian Dumitrescu @ 2021-03-15 22:48 UTC (permalink / raw) To: dev Meter arrays are stateful objects that are updated by the data plane and configured & monitored by the control plane. The meters implement the RFC 2698 Two Rate Three Color Marker (trTCM) algorithm. Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com> --- examples/pipeline/cli.c | 471 ++++++ examples/pipeline/examples/meter.cli | 31 + examples/pipeline/examples/meter.spec | 63 + lib/librte_pipeline/rte_swx_ctl.h | 155 ++ lib/librte_pipeline/rte_swx_pipeline.c | 1684 ++++++++++++++++--- lib/librte_pipeline/rte_swx_pipeline.h | 22 + lib/librte_pipeline/rte_swx_pipeline_spec.c | 91 + lib/librte_pipeline/version.map | 7 + 8 files changed, 2328 insertions(+), 196 deletions(-) create mode 100644 examples/pipeline/examples/meter.cli create mode 100644 examples/pipeline/examples/meter.spec diff --git a/examples/pipeline/cli.c b/examples/pipeline/cli.c index cdf8f13f3..045525307 100644 --- a/examples/pipeline/cli.c +++ b/examples/pipeline/cli.c @@ -1108,6 +1108,398 @@ cmd_pipeline_regwr(char **tokens, } } +static const char cmd_pipeline_meter_profile_add_help[] = +"pipeline <pipeline_name> meter profile <profile_name> add " + "cir <cir> pir <pir> cbs <cbs> pbs <pbs>\n"; + +static void +cmd_pipeline_meter_profile_add(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct rte_meter_trtcm_params params; + struct pipeline *p; + const char *profile_name; + int status; + + if (n_tokens != 14) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + if (strcmp(tokens[3], "profile")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile"); + return; + } + + profile_name = tokens[4]; + + if (strcmp(tokens[5], "add")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "add"); + return; + } + + if (strcmp(tokens[6], "cir")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cir"); + return; + } + + if (parser_read_uint64(¶ms.cir, tokens[7])) { + snprintf(out, out_size, MSG_ARG_INVALID, "cir"); + return; + } + + if (strcmp(tokens[8], "pir")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pir"); + return; + } + + if (parser_read_uint64(¶ms.pir, tokens[9])) { + snprintf(out, out_size, MSG_ARG_INVALID, "pir"); + return; + } + + if (strcmp(tokens[10], "cbs")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cbs"); + return; + } + + if (parser_read_uint64(¶ms.cbs, tokens[11])) { + snprintf(out, out_size, MSG_ARG_INVALID, "cbs"); + return; + } + + if (strcmp(tokens[12], "pbs")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pbs"); + return; + } + + if (parser_read_uint64(¶ms.pbs, tokens[13])) { + snprintf(out, out_size, MSG_ARG_INVALID, "pbs"); + return; + } + + status = rte_swx_ctl_meter_profile_add(p->p, profile_name, ¶ms); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + +static const char cmd_pipeline_meter_profile_delete_help[] = +"pipeline <pipeline_name> meter profile <profile_name> delete\n"; + +static void +cmd_pipeline_meter_profile_delete(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *profile_name; + int status; + + if (n_tokens != 6) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + if (strcmp(tokens[3], "profile")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile"); + return; + } + + profile_name = tokens[4]; + + if (strcmp(tokens[5], "delete")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "delete"); + return; + } + + status = rte_swx_ctl_meter_profile_delete(p->p, profile_name); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + +static const char cmd_pipeline_meter_reset_help[] = +"pipeline <pipeline_name> meter <meter_array_name> from <index0> to <index1> " + "reset\n"; + +static void +cmd_pipeline_meter_reset(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint32_t idx0, idx1; + + if (n_tokens != 9) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + name = tokens[3]; + + if (strcmp(tokens[4], "from")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "from"); + return; + } + + if (parser_read_uint32(&idx0, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index0"); + return; + } + + if (strcmp(tokens[6], "to")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "to"); + return; + } + + if (parser_read_uint32(&idx1, tokens[7]) || (idx1 < idx0)) { + snprintf(out, out_size, MSG_ARG_INVALID, "index1"); + return; + } + + if (strcmp(tokens[8], "reset")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "reset"); + return; + } + + for ( ; idx0 <= idx1; idx0++) { + int status; + + status = rte_swx_ctl_meter_reset(p->p, name, idx0); + if (status) { + snprintf(out, out_size, "Command failed for index %u.\n", idx0); + return; + } + } +} + +static const char cmd_pipeline_meter_set_help[] = +"pipeline <pipeline_name> meter <meter_array_name> from <index0> to <index1> " + "set profile <profile_name>\n"; + +static void +cmd_pipeline_meter_set(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name, *profile_name; + uint32_t idx0, idx1; + + if (n_tokens != 11) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + name = tokens[3]; + + if (strcmp(tokens[4], "from")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "from"); + return; + } + + if (parser_read_uint32(&idx0, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index0"); + return; + } + + if (strcmp(tokens[6], "to")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "to"); + return; + } + + if (parser_read_uint32(&idx1, tokens[7]) || (idx1 < idx0)) { + snprintf(out, out_size, MSG_ARG_INVALID, "index1"); + return; + } + + if (strcmp(tokens[8], "set")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "set"); + return; + } + + if (strcmp(tokens[9], "profile")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile"); + return; + } + + profile_name = tokens[10]; + + for ( ; idx0 <= idx1; idx0++) { + int status; + + status = rte_swx_ctl_meter_set(p->p, name, idx0, profile_name); + if (status) { + snprintf(out, out_size, "Command failed for index %u.\n", idx0); + return; + } + } +} + +static const char cmd_pipeline_meter_stats_help[] = +"pipeline <pipeline_name> meter <meter_array_name> from <index0> to <index1> " + "stats\n"; + +static void +cmd_pipeline_meter_stats(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct rte_swx_ctl_meter_stats stats; + struct pipeline *p; + const char *name; + uint32_t idx0, idx1; + + if (n_tokens != 9) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + name = tokens[3]; + + if (strcmp(tokens[4], "from")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "from"); + return; + } + + if (parser_read_uint32(&idx0, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index0"); + return; + } + + if (strcmp(tokens[6], "to")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "to"); + return; + } + + if (parser_read_uint32(&idx1, tokens[7]) || (idx1 < idx0)) { + snprintf(out, out_size, MSG_ARG_INVALID, "index1"); + return; + } + + if (strcmp(tokens[8], "stats")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "stats"); + return; + } + + /* Table header. */ + snprintf(out, out_size, "+-%7s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+\n", + "-------", + "----------------", "----------------", "----------------", + "----------------", "----------------", "----------------"); + out_size -= strlen(out); + out += strlen(out); + + snprintf(out, out_size, "| %4s | %16s | %16s | %16s | %16s | %16s | %16s |\n", + "METER #", + "GREEN (packets)", "YELLOW (packets)", "RED (packets)", + "GREEN (bytes)", "YELLOW (bytes)", "RED (bytes)"); + out_size -= strlen(out); + out += strlen(out); + + snprintf(out, out_size, "+-%7s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+\n", + "-------", + "----------------", "----------------", "----------------", + "----------------", "----------------", "----------------"); + out_size -= strlen(out); + out += strlen(out); + + /* Table rows. */ + for ( ; idx0 <= idx1; idx0++) { + int status; + + status = rte_swx_ctl_meter_stats_read(p->p, name, idx0, &stats); + if (status) { + snprintf(out, out_size, "Pipeline meter stats error at index %u.\n", idx0); + out_size -= strlen(out); + out += strlen(out); + return; + } + + snprintf(out, out_size, "| %7d | %16" PRIx64 " | %16" PRIx64 " | %16" PRIx64 + " | %16" PRIx64 " | %16" PRIx64 " | %16" PRIx64 " |\n", + idx0, + stats.n_pkts[RTE_COLOR_GREEN], + stats.n_pkts[RTE_COLOR_YELLOW], + stats.n_pkts[RTE_COLOR_RED], + stats.n_bytes[RTE_COLOR_GREEN], + stats.n_bytes[RTE_COLOR_YELLOW], + stats.n_bytes[RTE_COLOR_RED]); + out_size -= strlen(out); + out += strlen(out); + } +} + static const char cmd_pipeline_stats_help[] = "pipeline <pipeline_name> stats\n"; @@ -1303,6 +1695,11 @@ cmd_help(char **tokens, "\tpipeline table update\n" "\tpipeline regrd\n" "\tpipeline regwr\n" + "\tpipeline meter profile add\n" + "\tpipeline meter profile delete\n" + "\tpipeline meter reset\n" + "\tpipeline meter set\n" + "\tpipeline meter stats\n" "\tpipeline stats\n" "\tthread pipeline enable\n" "\tthread pipeline disable\n\n"); @@ -1373,6 +1770,43 @@ cmd_help(char **tokens, return; } + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 4) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[1], "profile") + && !strcmp(tokens[1], "add")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_profile_add_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 4) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[1], "profile") + && !strcmp(tokens[1], "delete")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_profile_delete_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 3) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "reset")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_reset_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 3) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "set")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_set_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 3) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "stats")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_stats_help); + return; + } + if ((n_tokens == 3) && (strcmp(tokens[0], "thread") == 0) && (strcmp(tokens[1], "pipeline") == 0)) { @@ -1481,6 +1915,43 @@ cli_process(char *in, char *out, size_t out_size, void *obj) return; } + if ((n_tokens >= 6) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[3], "profile") == 0) && + (strcmp(tokens[5], "add") == 0)) { + cmd_pipeline_meter_profile_add(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 6) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[3], "profile") == 0) && + (strcmp(tokens[5], "delete") == 0)) { + cmd_pipeline_meter_profile_delete(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 9) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[8], "reset") == 0)) { + cmd_pipeline_meter_reset(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 9) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[8], "set") == 0)) { + cmd_pipeline_meter_set(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 9) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[8], "stats") == 0)) { + cmd_pipeline_meter_stats(tokens, n_tokens, out, out_size, obj); + return; + } + if ((n_tokens >= 3) && (strcmp(tokens[2], "stats") == 0)) { cmd_pipeline_stats(tokens, n_tokens, out, out_size, diff --git a/examples/pipeline/examples/meter.cli b/examples/pipeline/examples/meter.cli new file mode 100644 index 000000000..b29ed2402 --- /dev/null +++ b/examples/pipeline/examples/meter.cli @@ -0,0 +1,31 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2020 Intel Corporation + +; Example command line: +; ./build/examples/dpdk-pipeline -l0-1 -- -s ./examples/pipeline/examples/meter.cli + +mempool MEMPOOL0 buffer 2304 pool 32K cache 256 cpu 0 + +link LINK0 dev 0000:18:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK1 dev 0000:18:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK2 dev 0000:3b:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK3 dev 0000:3b:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on + +pipeline PIPELINE0 create 0 + +pipeline PIPELINE0 port in 0 link LINK0 rxq 0 bsz 32 +pipeline PIPELINE0 port in 1 link LINK1 rxq 0 bsz 32 +pipeline PIPELINE0 port in 2 link LINK2 rxq 0 bsz 32 +pipeline PIPELINE0 port in 3 link LINK3 rxq 0 bsz 32 + +pipeline PIPELINE0 port out 0 link LINK0 txq 0 bsz 32 +pipeline PIPELINE0 port out 1 link LINK1 txq 0 bsz 32 +pipeline PIPELINE0 port out 2 link LINK2 txq 0 bsz 32 +pipeline PIPELINE0 port out 3 link LINK3 txq 0 bsz 32 + +pipeline PIPELINE0 build ./examples/pipeline/examples/meter.spec + +pipeline PIPELINE0 meter profile platinum add cir 46000000 pir 138000000 cbs 1000000 pbs 1000000 +pipeline PIPELINE0 meter meters from 0 to 15 set profile platinum + +thread 1 pipeline PIPELINE0 enable diff --git a/examples/pipeline/examples/meter.spec b/examples/pipeline/examples/meter.spec new file mode 100644 index 000000000..43b8170d7 --- /dev/null +++ b/examples/pipeline/examples/meter.spec @@ -0,0 +1,63 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2021 Intel Corporation + +; This program is setting up an array of Two Rate Three Color Marker (trTCM) meters called "meters". +; Every input packet (Ethernet/IPv4) is metered by the meter at the location indexed by the IPv4 +; header "Source Address" field. All green packets are sent out on port 0, the yellow ones on port 1 +; and the red ones on port 2. +; +; The "meter stats" CLI command can be used to read the packet and byte statistics counters of any +; meter in the array. + +// +// Headers. +// +struct ethernet_h { + bit<48> dst_addr + bit<48> src_addr + bit<16> ethertype +} + +struct ipv4_h { + bit<8> ver_ihl + bit<8> diffserv + bit<16> total_len + bit<16> identification + bit<16> flags_offset + bit<8> ttl + bit<8> protocol + bit<16> hdr_checksum + bit<32> src_addr + bit<32> dst_addr +} + +header ethernet instanceof ethernet_h +header ipv4 instanceof ipv4_h + +// +// Meta-data. +// +struct metadata_t { + bit<32> port_in + bit<32> port_out +} + +metadata instanceof metadata_t + +// +// Registers. +// +metarray meters size 65536 + +// +// Pipeline. +// +apply { + rx m.port_in + extract h.ethernet + extract h.ipv4 + meter meters h.ipv4.src_addr h.ipv4.total_len 0 m.port_out + emit h.ethernet + emit h.ipv4 + tx m.port_out +} diff --git a/lib/librte_pipeline/rte_swx_ctl.h b/lib/librte_pipeline/rte_swx_ctl.h index c0a55c6f0..b8f22c516 100644 --- a/lib/librte_pipeline/rte_swx_ctl.h +++ b/lib/librte_pipeline/rte_swx_ctl.h @@ -18,6 +18,7 @@ extern "C" { #include <stdio.h> #include <rte_compat.h> +#include <rte_meter.h> #include "rte_swx_port.h" #include "rte_swx_table.h" @@ -49,6 +50,9 @@ struct rte_swx_ctl_pipeline_info { /** Number of register arrays. */ uint32_t n_regarrays; + + /** Number of meter arrays. */ + uint32_t n_metarrays; }; /** @@ -636,6 +640,157 @@ rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, uint32_t regarray_index, uint64_t value); +/* + * Meter Array Query and Configuration API. + */ + +/** Meter array info. */ +struct rte_swx_ctl_metarray_info { + /** Meter array name. */ + char name[RTE_SWX_CTL_NAME_SIZE]; + + /** Meter array size. */ + uint32_t size; +}; + +/** + * Meter array info get + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_id + * Meter array ID (0 .. *n_metarrays* - 1). + * @param[out] metarray + * Meter array info. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p, + uint32_t metarray_id, + struct rte_swx_ctl_metarray_info *metarray); + +/** + * Meter profile add + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Meter profile name. + * @param[in] params + * Meter profile parameters. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Meter profile with this name already exists. + */ +__rte_experimental +int +rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p, + const char *name, + struct rte_meter_trtcm_params *params); + +/** + * Meter profile delete + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Meter profile name. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -EBUSY: Meter profile is currently in use. + */ +__rte_experimental +int +rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p, + const char *name); + +/** + * Meter reset + * + * Reset a meter within a given meter array to use the default profile that + * causes all the input packets to be colored as green. It is the responsibility + * of the control plane to make sure this meter is not used by the data plane + * pipeline before calling this function. + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_name + * Meter array name. + * @param[in] metarray_index + * Meter index within the meter array. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index); + +/** + * Meter set + * + * Set a meter within a given meter array to use a specific profile. It is the + * responsibility of the control plane to make sure this meter is not used by + * the data plane pipeline before calling this function. + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_name + * Meter array name. + * @param[in] metarray_index + * Meter index within the meter array. + * @param[in] profile_name + * Existing meter profile name. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_meter_set(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + const char *profile_name); + +/** Meter statistics counters. */ +struct rte_swx_ctl_meter_stats { + /** Number of packets tagged by the meter for each color. */ + uint64_t n_pkts[RTE_COLORS]; + + /** Number of bytes tagged by the meter for each color. */ + uint64_t n_bytes[RTE_COLORS]; +}; + +/** + * Meter statistics counters read + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_name + * Meter array name. + * @param[in] metarray_index + * Meter index within the meter array. + * @param[out] stats + * Meter statistics counters. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + struct rte_swx_ctl_meter_stats *stats); + /** * Pipeline control free * diff --git a/lib/librte_pipeline/rte_swx_pipeline.c b/lib/librte_pipeline/rte_swx_pipeline.c index 5be87ca4f..5257a5360 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.c +++ b/lib/librte_pipeline/rte_swx_pipeline.c @@ -12,6 +12,8 @@ #include <rte_common.h> #include <rte_prefetch.h> #include <rte_byteorder.h> +#include <rte_cycles.h> +#include <rte_meter.h> #include "rte_swx_pipeline.h" #include "rte_swx_ctl.h" @@ -458,6 +460,31 @@ enum instruction_type { INSTR_REGADD_RIM, /* index = I, src = MEFT */ INSTR_REGADD_RII, /* index = I, src = I */ + /* metprefetch METARRAY index + * prefetch METARRAY[index] + * index = HMEFTI + */ + INSTR_METPREFETCH_H, /* index = H */ + INSTR_METPREFETCH_M, /* index = MEFT */ + INSTR_METPREFETCH_I, /* index = I */ + + /* meter METARRAY index length color_in color_out + * color_out = meter(METARRAY[index], length, color_in) + * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF + */ + INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */ + INSTR_METER_HHI, /* index = H, length = H, color_in = I */ + INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */ + INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */ + INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */ + INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */ + INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */ + INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */ + INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */ + INSTR_METER_IHI, /* index = I, length = H, color_in = I */ + INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */ + INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */ + /* table TABLE */ INSTR_TABLE, @@ -607,6 +634,25 @@ struct instr_regarray { }; }; +struct instr_meter { + uint8_t metarray_id; + uint8_t pad[3]; + + union { + struct instr_operand idx; + uint32_t idx_val; + }; + + struct instr_operand length; + + union { + struct instr_operand color_in; + uint32_t color_in_val; + }; + + struct instr_operand color_out; +}; + struct instr_dma { struct { uint8_t header_id[8]; @@ -642,6 +688,7 @@ struct instruction { struct instr_hdr_validity valid; struct instr_dst_src mov; struct instr_regarray regarray; + struct instr_meter meter; struct instr_dma dma; struct instr_dst_src alu; struct instr_table table; @@ -739,6 +786,43 @@ struct regarray_runtime { uint32_t size_mask; }; +/* + * Meter array. + */ +struct meter_profile { + TAILQ_ENTRY(meter_profile) node; + char name[RTE_SWX_NAME_SIZE]; + struct rte_meter_trtcm_params params; + struct rte_meter_trtcm_profile profile; + uint32_t n_users; +}; + +TAILQ_HEAD(meter_profile_tailq, meter_profile); + +struct metarray { + TAILQ_ENTRY(metarray) node; + char name[RTE_SWX_NAME_SIZE]; + uint32_t size; + uint32_t id; +}; + +TAILQ_HEAD(metarray_tailq, metarray); + +struct meter { + struct rte_meter_trtcm m; + struct meter_profile *profile; + enum rte_color color_mask; + uint8_t pad[20]; + + uint64_t n_pkts[RTE_COLORS]; + uint64_t n_bytes[RTE_COLORS]; +}; + +struct metarray_runtime { + struct meter *metarray; + uint32_t size_mask; +}; + /* * Pipeline. */ @@ -1115,12 +1199,15 @@ struct rte_swx_pipeline { struct table_type_tailq table_types; struct table_tailq tables; struct regarray_tailq regarrays; + struct meter_profile_tailq meter_profiles; + struct metarray_tailq metarrays; struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; struct rte_swx_table_state *table_state; struct regarray_runtime *regarray_runtime; + struct metarray_runtime *metarray_runtime; struct instruction *instructions; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; @@ -1132,6 +1219,7 @@ struct rte_swx_pipeline { uint32_t n_actions; uint32_t n_tables; uint32_t n_regarrays; + uint32_t n_metarrays; uint32_t n_headers; uint32_t thread_id; uint32_t port_id; @@ -5658,255 +5746,1078 @@ instr_regadd_rii_exec(struct rte_swx_pipeline *p) } /* - * jmp. + * metarray. */ -static struct action * -action_find(struct rte_swx_pipeline *p, const char *name); - -static int -instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(n_tokens == 2, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - instr->type = INSTR_JMP; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} - -static int -instr_jmp_valid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct header *h; - - CHECK(n_tokens == 3, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); - - instr->type = INSTR_JMP_VALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; - return 0; -} +static struct metarray * +metarray_find(struct rte_swx_pipeline *p, const char *name); static int -instr_jmp_invalid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, +instr_metprefetch_translate(struct rte_swx_pipeline *p, + struct action *action, char **tokens, int n_tokens, struct instruction *instr, - struct instruction_data *data) + struct instruction_data *data __rte_unused) { - struct header *h; + char *metarray = tokens[1], *idx = tokens[2]; + struct metarray *m; + struct field *fidx; + uint32_t idx_struct_id, idx_val; CHECK(n_tokens == 3, EINVAL); - strcpy(data->jmp_label, tokens[1]); + m = metarray_find(p, metarray); + CHECK(m, EINVAL); - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + /* METPREFETCH_H, METPREFETCH_M. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_METPREFETCH_M; + if (idx[0] == 'h') + instr->type = INSTR_METPREFETCH_H; - instr->type = INSTR_JMP_INVALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; + instr->meter.metarray_id = m->id; + instr->meter.idx.struct_id = (uint8_t)idx_struct_id; + instr->meter.idx.n_bits = fidx->n_bits; + instr->meter.idx.offset = fidx->offset / 8; + return 0; + } + + /* METPREFETCH_I. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_METPREFETCH_I; + instr->meter.metarray_id = m->id; + instr->meter.idx_val = idx_val; return 0; } static int -instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_meter_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3]; + char *color_in = tokens[4], *color_out = tokens[5]; + struct metarray *m; + struct field *fidx, *flength, *fcin, *fcout; + uint32_t idx_struct_id, length_struct_id; + uint32_t color_in_struct_id, color_out_struct_id; - strcpy(data->jmp_label, tokens[1]); + CHECK(n_tokens == 6, EINVAL); - instr->type = INSTR_JMP_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + m = metarray_find(p, metarray); + CHECK(m, EINVAL); -static int -instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + fidx = struct_field_parse(p, action, idx, &idx_struct_id); - strcpy(data->jmp_label, tokens[1]); + flength = struct_field_parse(p, action, length, &length_struct_id); + CHECK(flength, EINVAL); - instr->type = INSTR_JMP_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + fcin = struct_field_parse(p, action, color_in, &color_in_struct_id); -static int -instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id); + CHECK(fcout, EINVAL); - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */ + if (fidx && fcin) { + instr->type = INSTR_METER_MMM; + if (idx[0] == 'h' && length[0] == 'h') + instr->type = INSTR_METER_HHM; + if (idx[0] == 'h' && length[0] != 'h') + instr->type = INSTR_METER_HMM; + if (idx[0] != 'h' && length[0] == 'h') + instr->type = INSTR_METER_MHM; - strcpy(data->jmp_label, tokens[1]); + instr->meter.metarray_id = m->id; - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + instr->meter.idx.struct_id = (uint8_t)idx_struct_id; + instr->meter.idx.n_bits = fidx->n_bits; + instr->meter.idx.offset = fidx->offset / 8; - instr->type = INSTR_JMP_ACTION_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; -static int -instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id; + instr->meter.color_in.n_bits = fcin->n_bits; + instr->meter.color_in.offset = fcin->offset / 8; - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; - strcpy(data->jmp_label, tokens[1]); + return 0; + } - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */ + if (fidx && !fcin) { + uint32_t color_in_val = strtoul(color_in, &color_in, 0); + CHECK(!color_in[0], EINVAL); - instr->type = INSTR_JMP_ACTION_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + instr->type = INSTR_METER_MMI; + if (idx[0] == 'h' && length[0] == 'h') + instr->type = INSTR_METER_HHI; + if (idx[0] == 'h' && length[0] != 'h') + instr->type = INSTR_METER_HMI; + if (idx[0] != 'h' && length[0] == 'h') + instr->type = INSTR_METER_MHI; -static int -instr_jmp_eq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + instr->meter.metarray_id = m->id; - CHECK(n_tokens == 4, EINVAL); + instr->meter.idx.struct_id = (uint8_t)idx_struct_id; + instr->meter.idx.n_bits = fidx->n_bits; + instr->meter.idx.offset = fidx->offset / 8; - strcpy(data->jmp_label, tokens[1]); + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + instr->meter.color_in_val = color_in_val; - /* JMP_EQ or JMP_EQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_EQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_EQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; return 0; } - /* JMP_EQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */ + if (!fidx && fcin) { + uint32_t idx_val; - if (a[0] == 'h') - b_val = hton64(b_val) >> (64 - fa->n_bits); + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - instr->type = INSTR_JMP_EQ_I; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b_val = b_val; - return 0; -} + instr->type = INSTR_METER_IMM; + if (length[0] == 'h') + instr->type = INSTR_METER_IHM; -static int -instr_jmp_neq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + instr->meter.metarray_id = m->id; - CHECK(n_tokens == 4, EINVAL); + instr->meter.idx_val = idx_val; - strcpy(data->jmp_label, tokens[1]); + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id; + instr->meter.color_in.n_bits = fcin->n_bits; + instr->meter.color_in.offset = fcin->offset / 8; - /* JMP_NEQ or JMP_NEQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_NEQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_NEQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; return 0; } - /* JMP_NEQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* index = I, length = HMEFT, color_in = I, color_out = MEF. */ + if (!fidx && !fcin) { + uint32_t idx_val, color_in_val; - if (a[0] == 'h') + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + color_in_val = strtoul(color_in, &color_in, 0); + CHECK(!color_in[0], EINVAL); + + instr->type = INSTR_METER_IMI; + if (length[0] == 'h') + instr->type = INSTR_METER_IHI; + + instr->meter.metarray_id = m->id; + + instr->meter.idx_val = idx_val; + + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; + + instr->meter.color_in_val = color_in_val; + + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; + + return 0; + } + + CHECK(0, EINVAL); +} + +static inline struct meter * +instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return &r->metarray[idx]; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline struct meter * +instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; + + return &r->metarray[idx]; +} + +#else + +#define instr_meter_idx_nbo instr_meter_idx_hbo + +#endif + +static inline struct meter * +instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint64_t idx = ip->meter.idx_val & r->size_mask; + + return &r->metarray[idx]; +} + +static inline uint32_t +instr_meter_length_hbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); + uint64_t src = src64 & src64_mask; + + return (uint32_t)src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint32_t +instr_meter_length_nbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); + + return (uint32_t)src; +} + +#else + +#define instr_meter_length_nbo instr_meter_length_hbo + +#endif + +static inline enum rte_color +instr_meter_color_in_hbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); + uint64_t src = src64 & src64_mask; + + return (enum rte_color)src; +} + +static inline void +instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out) +{ + uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); + + uint64_t src = (uint64_t)color_out; + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +static inline void +instr_metprefetch_h_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + + TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_metprefetch_m_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + + TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_metprefetch_i_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + + TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_hhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_hhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_hmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} +static inline void +instr_meter_hmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_ihm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_ihi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_imm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} +static inline void +instr_meter_imi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +/* + * jmp. + */ +static struct action * +action_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_valid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_VALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_invalid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_INVALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_eq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_EQ or JMP_EQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_EQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_EQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_EQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = hton64(b_val) >> (64 - fa->n_bits); + + instr->type = INSTR_JMP_EQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = b_val; + return 0; +} + +static int +instr_jmp_neq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_NEQ or JMP_NEQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_NEQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_NEQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_NEQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') b_val = hton64(b_val) >> (64 - fa->n_bits); instr->type = INSTR_JMP_NEQ_I; @@ -6543,6 +7454,22 @@ instr_translate(struct rte_swx_pipeline *p, instr, data); + if (!strcmp(tokens[tpos], "metprefetch")) + return instr_metprefetch_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "meter")) + return instr_meter_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + if (!strcmp(tokens[tpos], "table")) return instr_table_translate(p, action, @@ -7210,6 +8137,23 @@ static instr_exec_t instruction_table[] = { [INSTR_REGADD_RIM] = instr_regadd_rim_exec, [INSTR_REGADD_RII] = instr_regadd_rii_exec, + [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec, + [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec, + [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec, + + [INSTR_METER_HHM] = instr_meter_hhm_exec, + [INSTR_METER_HHI] = instr_meter_hhi_exec, + [INSTR_METER_HMM] = instr_meter_hmm_exec, + [INSTR_METER_HMI] = instr_meter_hmi_exec, + [INSTR_METER_MHM] = instr_meter_mhm_exec, + [INSTR_METER_MHI] = instr_meter_mhi_exec, + [INSTR_METER_MMM] = instr_meter_mmm_exec, + [INSTR_METER_MMI] = instr_meter_mmi_exec, + [INSTR_METER_IHM] = instr_meter_ihm_exec, + [INSTR_METER_IHI] = instr_meter_ihi_exec, + [INSTR_METER_IMM] = instr_meter_imm_exec, + [INSTR_METER_IMI] = instr_meter_imi_exec, + [INSTR_TABLE] = instr_table_exec, [INSTR_EXTERN_OBJ] = instr_extern_obj_exec, [INSTR_EXTERN_FUNC] = instr_extern_func_exec, @@ -8048,6 +8992,182 @@ regarray_free(struct rte_swx_pipeline *p) } } +/* + * Meter array. + */ +static struct meter_profile * +meter_profile_find(struct rte_swx_pipeline *p, const char *name) +{ + struct meter_profile *elem; + + TAILQ_FOREACH(elem, &p->meter_profiles, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct metarray * +metarray_find(struct rte_swx_pipeline *p, const char *name) +{ + struct metarray *elem; + + TAILQ_FOREACH(elem, &p->metarrays, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct metarray * +metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id) +{ + struct metarray *elem = NULL; + + TAILQ_FOREACH(elem, &p->metarrays, node) + if (elem->id == id) + return elem; + + return NULL; +} + +int +rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size) +{ + struct metarray *m; + + CHECK(p, EINVAL); + + CHECK_NAME(name, EINVAL); + CHECK(!metarray_find(p, name), EEXIST); + + CHECK(size, EINVAL); + size = rte_align32pow2(size); + + /* Memory allocation. */ + m = calloc(1, sizeof(struct metarray)); + CHECK(m, ENOMEM); + + /* Node initialization. */ + strcpy(m->name, name); + m->size = size; + m->id = p->n_metarrays; + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->metarrays, m, node); + p->n_metarrays++; + + return 0; +} + +struct meter_profile meter_profile_default = { + .node = {0}, + .name = "", + .params = {0}, + + .profile = { + .cbs = 10000, + .pbs = 10000, + .cir_period = 1, + .cir_bytes_per_period = 1, + .pir_period = 1, + .pir_bytes_per_period = 1, + }, + + .n_users = 0, +}; + +static void +meter_init(struct meter *m) +{ + memset(m, 0, sizeof(struct meter)); + rte_meter_trtcm_config(&m->m, &meter_profile_default.profile); + m->profile = &meter_profile_default; + m->color_mask = RTE_COLOR_GREEN; + + meter_profile_default.n_users++; +} + +static int +metarray_build(struct rte_swx_pipeline *p) +{ + struct metarray *m; + + if (!p->n_metarrays) + return 0; + + p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime)); + CHECK(p->metarray_runtime, ENOMEM); + + TAILQ_FOREACH(m, &p->metarrays, node) { + struct metarray_runtime *r = &p->metarray_runtime[m->id]; + uint32_t i; + + r->metarray = env_malloc(m->size * sizeof(struct meter), + RTE_CACHE_LINE_SIZE, + p->numa_node); + CHECK(r->metarray, ENOMEM); + + for (i = 0; i < m->size; i++) + meter_init(&r->metarray[i]); + + r->size_mask = m->size - 1; + } + + return 0; +} + +static void +metarray_build_free(struct rte_swx_pipeline *p) +{ + uint32_t i; + + if (!p->metarray_runtime) + return; + + for (i = 0; i < p->n_metarrays; i++) { + struct metarray *m = metarray_find_by_id(p, i); + struct metarray_runtime *r = &p->metarray_runtime[i]; + + env_free(r->metarray, m->size * sizeof(struct meter)); + } + + free(p->metarray_runtime); + p->metarray_runtime = NULL; +} + +static void +metarray_free(struct rte_swx_pipeline *p) +{ + metarray_build_free(p); + + /* Meter arrays. */ + for ( ; ; ) { + struct metarray *elem; + + elem = TAILQ_FIRST(&p->metarrays); + if (!elem) + break; + + TAILQ_REMOVE(&p->metarrays, elem, node); + free(elem); + } + + /* Meter profiles. */ + for ( ; ; ) { + struct meter_profile *elem; + + elem = TAILQ_FIRST(&p->meter_profiles); + if (!elem) + break; + + TAILQ_REMOVE(&p->meter_profiles, elem, node); + free(elem); + } +} + /* * Pipeline. */ @@ -8077,6 +9197,8 @@ rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node) TAILQ_INIT(&pipeline->table_types); TAILQ_INIT(&pipeline->tables); TAILQ_INIT(&pipeline->regarrays); + TAILQ_INIT(&pipeline->meter_profiles); + TAILQ_INIT(&pipeline->metarrays); pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */ pipeline->numa_node = numa_node; @@ -8093,6 +9215,7 @@ rte_swx_pipeline_free(struct rte_swx_pipeline *p) free(p->instructions); + metarray_free(p); regarray_free(p); table_state_free(p); table_free(p); @@ -8182,10 +9305,15 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p) if (status) goto error; + status = metarray_build(p); + if (status) + goto error; + p->build_done = 1; return 0; error: + metarray_build_free(p); regarray_build_free(p); table_state_build_free(p); table_build_free(p); @@ -8248,6 +9376,7 @@ rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p, pipeline->n_actions = n_actions; pipeline->n_tables = n_tables; pipeline->n_regarrays = p->n_regarrays; + pipeline->n_metarrays = p->n_metarrays; return 0; } @@ -8516,3 +9645,166 @@ rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, r->regarray[regarray_index] = value; return 0; } + +int +rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p, + uint32_t metarray_id, + struct rte_swx_ctl_metarray_info *metarray) +{ + struct metarray *m; + + if (!p || !metarray) + return -EINVAL; + + m = metarray_find_by_id(p, metarray_id); + if (!m) + return -EINVAL; + + strcpy(metarray->name, m->name); + metarray->size = m->size; + return 0; +} + +int +rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p, + const char *name, + struct rte_meter_trtcm_params *params) +{ + struct meter_profile *mp; + int status; + + CHECK(p, EINVAL); + CHECK_NAME(name, EINVAL); + CHECK(params, EINVAL); + CHECK(!meter_profile_find(p, name), EEXIST); + + /* Node allocation. */ + mp = calloc(1, sizeof(struct meter_profile)); + CHECK(mp, ENOMEM); + + /* Node initialization. */ + strcpy(mp->name, name); + memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params)); + status = rte_meter_trtcm_profile_config(&mp->profile, params); + if (status) { + free(mp); + CHECK(0, EINVAL); + } + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node); + + return 0; +} + +int +rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p, + const char *name) +{ + struct meter_profile *mp; + + CHECK(p, EINVAL); + CHECK_NAME(name, EINVAL); + + mp = meter_profile_find(p, name); + CHECK(mp, EINVAL); + CHECK(!mp->n_users, EBUSY); + + /* Remove node from tailq. */ + TAILQ_REMOVE(&p->meter_profiles, mp, node); + free(mp); + + return 0; +} + +int +rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index) +{ + struct meter_profile *mp_old; + struct metarray *metarray; + struct metarray_runtime *metarray_runtime; + struct meter *m; + + CHECK(p, EINVAL); + CHECK_NAME(metarray_name, EINVAL); + + metarray = metarray_find(p, metarray_name); + CHECK(metarray, EINVAL); + CHECK(metarray_index < metarray->size, EINVAL); + + metarray_runtime = &p->metarray_runtime[metarray->id]; + m = &metarray_runtime->metarray[metarray_index]; + mp_old = m->profile; + + meter_init(m); + + mp_old->n_users--; + + return 0; +} + +int +rte_swx_ctl_meter_set(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + const char *profile_name) +{ + struct meter_profile *mp, *mp_old; + struct metarray *metarray; + struct metarray_runtime *metarray_runtime; + struct meter *m; + + CHECK(p, EINVAL); + CHECK_NAME(metarray_name, EINVAL); + + metarray = metarray_find(p, metarray_name); + CHECK(metarray, EINVAL); + CHECK(metarray_index < metarray->size, EINVAL); + + mp = meter_profile_find(p, profile_name); + CHECK(mp, EINVAL); + + metarray_runtime = &p->metarray_runtime[metarray->id]; + m = &metarray_runtime->metarray[metarray_index]; + mp_old = m->profile; + + memset(m, 0, sizeof(struct meter)); + rte_meter_trtcm_config(&m->m, &mp->profile); + m->profile = mp; + m->color_mask = RTE_COLORS; + + mp->n_users++; + mp_old->n_users--; + + return 0; +} + +int +rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + struct rte_swx_ctl_meter_stats *stats) +{ + struct metarray *metarray; + struct metarray_runtime *metarray_runtime; + struct meter *m; + + CHECK(p, EINVAL); + CHECK_NAME(metarray_name, EINVAL); + + metarray = metarray_find(p, metarray_name); + CHECK(metarray, EINVAL); + CHECK(metarray_index < metarray->size, EINVAL); + + CHECK(stats, EINVAL); + + metarray_runtime = &p->metarray_runtime[metarray->id]; + m = &metarray_runtime->metarray[metarray_index]; + + memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts)); + memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes)); + + return 0; +} diff --git a/lib/librte_pipeline/rte_swx_pipeline.h b/lib/librte_pipeline/rte_swx_pipeline.h index 1c9b2eb12..156b1fd67 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.h +++ b/lib/librte_pipeline/rte_swx_pipeline.h @@ -640,6 +640,28 @@ rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, uint32_t size, uint64_t init_val); +/** + * Pipeline meter array configure + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Meter array name. + * @param[in] size + * Number of meters in the array. Each meter in the array implements the Two + * Rate Three Color Marker (trTCM) algorithm, as specified by RFC 2698. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Meter array with this name already exists. + */ +__rte_experimental +int +rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size); + /** * Pipeline instructions configure * diff --git a/lib/librte_pipeline/rte_swx_pipeline_spec.c b/lib/librte_pipeline/rte_swx_pipeline_spec.c index 0d0bca2ed..2e867d7bf 100644 --- a/lib/librte_pipeline/rte_swx_pipeline_spec.c +++ b/lib/librte_pipeline/rte_swx_pipeline_spec.c @@ -1015,6 +1015,68 @@ regarray_statement_parse(struct regarray_spec *s, return 0; } +/* + * metarray. + * + * metarray NAME size SIZE + */ +struct metarray_spec { + char *name; + uint32_t size; +}; + +static void +metarray_spec_free(struct metarray_spec *s) +{ + if (!s) + return; + + free(s->name); + s->name = NULL; +} + +static int +metarray_statement_parse(struct metarray_spec *s, + char **tokens, + uint32_t n_tokens, + uint32_t n_lines, + uint32_t *err_line, + const char **err_msg) +{ + char *p; + + /* Check format. */ + if ((n_tokens != 4) || strcmp(tokens[2], "size")) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid metarray statement."; + return -EINVAL; + } + + /* spec. */ + s->name = strdup(tokens[1]); + if (!s->name) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Memory allocation failed."; + return -ENOMEM; + } + + p = tokens[3]; + s->size = strtoul(p, &p, 0); + if (p[0] || !s->size) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid size argument."; + return -EINVAL; + } + + return 0; +} + /* * apply. * @@ -1142,6 +1204,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, struct action_spec action_spec = {0}; struct table_spec table_spec = {0}; struct regarray_spec regarray_spec = {0}; + struct metarray_spec metarray_spec = {0}; struct apply_spec apply_spec = {0}; uint32_t n_lines; uint32_t block_mask = 0; @@ -1509,6 +1572,33 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, continue; } + /* metarray. */ + if (!strcmp(tokens[0], "metarray")) { + status = metarray_statement_parse(&metarray_spec, + tokens, + n_tokens, + n_lines, + err_line, + err_msg); + if (status) + goto error; + + status = rte_swx_pipeline_metarray_config(p, + metarray_spec.name, + metarray_spec.size); + if (status) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Meter array configuration error."; + goto error; + } + + metarray_spec_free(&metarray_spec); + + continue; + } + /* apply. */ if (!strcmp(tokens[0], "apply")) { status = apply_statement_parse(&block_mask, @@ -1562,6 +1652,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, action_spec_free(&action_spec); table_spec_free(&table_spec); regarray_spec_free(®array_spec); + metarray_spec_free(&metarray_spec); apply_spec_free(&apply_spec); return status; } diff --git a/lib/librte_pipeline/version.map b/lib/librte_pipeline/version.map index 26a9edf47..2049d6d69 100644 --- a/lib/librte_pipeline/version.map +++ b/lib/librte_pipeline/version.map @@ -104,8 +104,15 @@ EXPERIMENTAL { rte_swx_pipeline_table_type_register; #added in 21.05 + rte_swx_ctl_metarray_info_get; + rte_swx_ctl_meter_profile_add; + rte_swx_ctl_meter_profile_delete; + rte_swx_ctl_meter_reset; + rte_swx_ctl_meter_set; + rte_swx_ctl_meter_stats_read; rte_swx_ctl_pipeline_regarray_read; rte_swx_ctl_pipeline_regarray_write; rte_swx_ctl_regarray_info_get; + rte_swx_pipeline_metarray_config; rte_swx_pipeline_regarray_config; }; -- 2.17.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* [dpdk-dev] [PATCH v5 1/2] pipeline: add register array support to SWX pipeline 2021-03-15 22:48 ` [dpdk-dev] [PATCH v4 1/2] pipeline: add register array support to " Cristian Dumitrescu 2021-03-15 22:48 ` [dpdk-dev] [PATCH v4 2/2] pipeline: add meter support to the " Cristian Dumitrescu @ 2021-03-16 11:40 ` Cristian Dumitrescu 2021-03-16 11:40 ` [dpdk-dev] [PATCH v5 2/2] pipeline: add meter support to the " Cristian Dumitrescu 1 sibling, 1 reply; 8+ messages in thread From: Cristian Dumitrescu @ 2021-03-16 11:40 UTC (permalink / raw) To: dev Register arrays are stateful objects that can be read & modified by both the data plane and the control plane, as opposed to tables, which are read-only for data plane. One key use-case is the implementation of stats counters. Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com> --- examples/pipeline/cli.c | 125 ++ examples/pipeline/examples/registers.cli | 28 + examples/pipeline/examples/registers.spec | 67 + lib/librte_pipeline/rte_swx_ctl.h | 79 + lib/librte_pipeline/rte_swx_pipeline.c | 1602 +++++++++++++++++-- lib/librte_pipeline/rte_swx_pipeline.h | 24 + lib/librte_pipeline/rte_swx_pipeline_spec.c | 105 ++ lib/librte_pipeline/version.map | 6 + 8 files changed, 1882 insertions(+), 154 deletions(-) create mode 100644 examples/pipeline/examples/registers.cli create mode 100644 examples/pipeline/examples/registers.spec diff --git a/examples/pipeline/cli.c b/examples/pipeline/cli.c index 30c2dd34d..cdf8f13f3 100644 --- a/examples/pipeline/cli.c +++ b/examples/pipeline/cli.c @@ -1009,6 +1009,105 @@ cmd_pipeline_table_update(char **tokens, fclose(file_default); } +static const char cmd_pipeline_regrd_help[] = +"pipeline <pipeline_name> regrd <register_array_name> <index>\n"; + +static void +cmd_pipeline_regrd(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 5) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regrd")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regrd"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_read(p->p, name, idx, &value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } + + snprintf(out, out_size, "0x%" PRIx64 "\n", value); +} + +static const char cmd_pipeline_regwr_help[] = +"pipeline <pipeline_name> regwr <register_array_name> <index> <value>\n"; + +static void +cmd_pipeline_regwr(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 6) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regwr")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regwr"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + if (parser_read_uint64(&value, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "value"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_write(p->p, name, idx, value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + static const char cmd_pipeline_stats_help[] = "pipeline <pipeline_name> stats\n"; @@ -1202,6 +1301,8 @@ cmd_help(char **tokens, "\tpipeline port out\n" "\tpipeline build\n" "\tpipeline table update\n" + "\tpipeline regrd\n" + "\tpipeline regwr\n" "\tpipeline stats\n" "\tthread pipeline enable\n" "\tthread pipeline disable\n\n"); @@ -1254,6 +1355,18 @@ cmd_help(char **tokens, return; } + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regrd") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regrd_help); + return; + } + + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regwr") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regwr_help); + return; + } + if ((strcmp(tokens[0], "pipeline") == 0) && (n_tokens == 2) && (strcmp(tokens[1], "stats") == 0)) { snprintf(out, out_size, "\n%s\n", cmd_pipeline_stats_help); @@ -1356,6 +1469,18 @@ cli_process(char *in, char *out, size_t out_size, void *obj) return; } + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regrd") == 0)) { + cmd_pipeline_regrd(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regwr") == 0)) { + cmd_pipeline_regwr(tokens, n_tokens, out, out_size, obj); + return; + } + if ((n_tokens >= 3) && (strcmp(tokens[2], "stats") == 0)) { cmd_pipeline_stats(tokens, n_tokens, out, out_size, diff --git a/examples/pipeline/examples/registers.cli b/examples/pipeline/examples/registers.cli new file mode 100644 index 000000000..8d026294c --- /dev/null +++ b/examples/pipeline/examples/registers.cli @@ -0,0 +1,28 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2020 Intel Corporation + +; Example command line: +; ./build/examples/dpdk-pipeline -l0-1 -- -s ./examples/pipeline/examples/registers.cli + +mempool MEMPOOL0 buffer 2304 pool 32K cache 256 cpu 0 + +link LINK0 dev 0000:18:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK1 dev 0000:18:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK2 dev 0000:3b:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK3 dev 0000:3b:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on + +pipeline PIPELINE0 create 0 + +pipeline PIPELINE0 port in 0 link LINK0 rxq 0 bsz 32 +pipeline PIPELINE0 port in 1 link LINK1 rxq 0 bsz 32 +pipeline PIPELINE0 port in 2 link LINK2 rxq 0 bsz 32 +pipeline PIPELINE0 port in 3 link LINK3 rxq 0 bsz 32 + +pipeline PIPELINE0 port out 0 link LINK0 txq 0 bsz 32 +pipeline PIPELINE0 port out 1 link LINK1 txq 0 bsz 32 +pipeline PIPELINE0 port out 2 link LINK2 txq 0 bsz 32 +pipeline PIPELINE0 port out 3 link LINK3 txq 0 bsz 32 + +pipeline PIPELINE0 build ./examples/pipeline/examples/registers.spec + +thread 1 pipeline PIPELINE0 enable diff --git a/examples/pipeline/examples/registers.spec b/examples/pipeline/examples/registers.spec new file mode 100644 index 000000000..74a014ad0 --- /dev/null +++ b/examples/pipeline/examples/registers.spec @@ -0,0 +1,67 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2021 Intel Corporation + +; This program is setting up two register arrays called "pkt_counters" and "byte_counters". +; On every input packet (Ethernet/IPv4), the "pkt_counters" register at location indexed by +; the IPv4 header "Source Address" field is incremented, while the same location in the +; "byte_counters" array accummulates the value of the IPv4 header "Total Length" field. +; +; The "regrd" and "regwr" CLI commands can be used to read and write the current value of +; any register array location. + +// +// Headers. +// +struct ethernet_h { + bit<48> dst_addr + bit<48> src_addr + bit<16> ethertype +} + +struct ipv4_h { + bit<8> ver_ihl + bit<8> diffserv + bit<16> total_len + bit<16> identification + bit<16> flags_offset + bit<8> ttl + bit<8> protocol + bit<16> hdr_checksum + bit<32> src_addr + bit<32> dst_addr +} + +header ethernet instanceof ethernet_h +header ipv4 instanceof ipv4_h + +// +// Meta-data. +// +struct metadata_t { + bit<32> port_in + bit<32> port_out +} + +metadata instanceof metadata_t + +// +// Registers. +// +regarray pkt_counters size 65536 initval 0 +regarray byte_counters size 65536 initval 0 + +// +// Pipeline. +// +apply { + rx m.port_in + extract h.ethernet + extract h.ipv4 + regadd pkt_counters h.ipv4.src_addr 1 + regadd byte_counters h.ipv4.src_addr h.ipv4.total_len + mov m.port_out m.port_in + xor m.port_out 1 + emit h.ethernet + emit h.ipv4 + tx m.port_out +} diff --git a/lib/librte_pipeline/rte_swx_ctl.h b/lib/librte_pipeline/rte_swx_ctl.h index 530671db1..c0a55c6f0 100644 --- a/lib/librte_pipeline/rte_swx_ctl.h +++ b/lib/librte_pipeline/rte_swx_ctl.h @@ -46,6 +46,9 @@ struct rte_swx_ctl_pipeline_info { /** Number of tables. */ uint32_t n_tables; + + /** Number of register arrays. */ + uint32_t n_regarrays; }; /** @@ -557,6 +560,82 @@ rte_swx_ctl_pipeline_table_fprintf(FILE *f, struct rte_swx_ctl_pipeline *ctl, const char *table_name); +/* + * Register Array Query API. + */ + +/** Register array info. */ +struct rte_swx_ctl_regarray_info { + /** Register array name. */ + char name[RTE_SWX_CTL_NAME_SIZE]; + + /** Register array size. */ + uint32_t size; +}; + +/** + * Register array info get + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_id + * Register array ID (0 .. *n_regarrays* - 1). + * @param[out] regarray + * Register array info. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray); + +/** + * Register read + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[out] value + * Current register value. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value); + +/** + * Register write + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[in] value + * Value to be written to the register. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value); + /** * Pipeline control free * diff --git a/lib/librte_pipeline/rte_swx_pipeline.c b/lib/librte_pipeline/rte_swx_pipeline.c index eaaed7a0a..5be87ca4f 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.c +++ b/lib/librte_pipeline/rte_swx_pipeline.c @@ -45,9 +45,59 @@ do { \ #define TRACE(...) #endif +/* + * Environment. + */ #define ntoh64(x) rte_be_to_cpu_64(x) #define hton64(x) rte_cpu_to_be_64(x) +#ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE + +#include <rte_malloc.h> + +static void * +env_malloc(size_t size, size_t alignment, int numa_node) +{ + return rte_zmalloc_socket(NULL, size, alignment, numa_node); +} + +static void +env_free(void *start, size_t size __rte_unused) +{ + rte_free(start); +} + +#else + +#include <numa.h> + +static void * +env_malloc(size_t size, size_t alignment __rte_unused, int numa_node) +{ + void *start; + + if (numa_available() == -1) + return NULL; + + start = numa_alloc_onnode(size, numa_node); + if (!start) + return NULL; + + memset(start, 0, size); + return start; +} + +static void +env_free(void *start, size_t size) +{ + if (numa_available() == -1) + return; + + numa_free(start, size); +} + +#endif + /* * Struct. */ @@ -361,6 +411,53 @@ enum instruction_type { INSTR_ALU_SHR_MI, /* dst = MEF, src = I */ INSTR_ALU_SHR_HI, /* dst = H, src = I */ + /* regprefetch REGARRAY index + * prefetch REGARRAY[index] + * index = HMEFTI + */ + INSTR_REGPREFETCH_RH, /* index = H */ + INSTR_REGPREFETCH_RM, /* index = MEFT */ + INSTR_REGPREFETCH_RI, /* index = I */ + + /* regrd dst REGARRAY index + * dst = REGARRAY[index] + * dst = HMEF, index = HMEFTI + */ + INSTR_REGRD_HRH, /* dst = H, index = H */ + INSTR_REGRD_HRM, /* dst = H, index = MEFT */ + INSTR_REGRD_HRI, /* dst = H, index = I */ + INSTR_REGRD_MRH, /* dst = MEF, index = H */ + INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */ + INSTR_REGRD_MRI, /* dst = MEF, index = I */ + + /* regwr REGARRAY index src + * REGARRAY[index] = src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGWR_RHH, /* index = H, src = H */ + INSTR_REGWR_RHM, /* index = H, src = MEFT */ + INSTR_REGWR_RHI, /* index = H, src = I */ + INSTR_REGWR_RMH, /* index = MEFT, src = H */ + INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGWR_RMI, /* index = MEFT, src = I */ + INSTR_REGWR_RIH, /* index = I, src = H */ + INSTR_REGWR_RIM, /* index = I, src = MEFT */ + INSTR_REGWR_RII, /* index = I, src = I */ + + /* regadd REGARRAY index src + * REGARRAY[index] += src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGADD_RHH, /* index = H, src = H */ + INSTR_REGADD_RHM, /* index = H, src = MEFT */ + INSTR_REGADD_RHI, /* index = H, src = I */ + INSTR_REGADD_RMH, /* index = MEFT, src = H */ + INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGADD_RMI, /* index = MEFT, src = I */ + INSTR_REGADD_RIH, /* index = I, src = H */ + INSTR_REGADD_RIM, /* index = I, src = MEFT */ + INSTR_REGADD_RII, /* index = I, src = I */ + /* table TABLE */ INSTR_TABLE, @@ -495,6 +592,21 @@ struct instr_dst_src { }; }; +struct instr_regarray { + uint8_t regarray_id; + uint8_t pad[3]; + + union { + struct instr_operand idx; + uint32_t idx_val; + }; + + union { + struct instr_operand dstsrc; + uint64_t dstsrc_val; + }; +}; + struct instr_dma { struct { uint8_t header_id[8]; @@ -529,6 +641,7 @@ struct instruction { struct instr_io io; struct instr_hdr_validity valid; struct instr_dst_src mov; + struct instr_regarray regarray; struct instr_dma dma; struct instr_dst_src alu; struct instr_table table; @@ -608,6 +721,24 @@ struct table_runtime { uint8_t **key; }; +/* + * Register array. + */ +struct regarray { + TAILQ_ENTRY(regarray) node; + char name[RTE_SWX_NAME_SIZE]; + uint64_t init_val; + uint32_t size; + uint32_t id; +}; + +TAILQ_HEAD(regarray_tailq, regarray); + +struct regarray_runtime { + uint64_t *regarray; + uint32_t size_mask; +}; + /* * Pipeline. */ @@ -983,11 +1114,13 @@ struct rte_swx_pipeline { struct action_tailq actions; struct table_type_tailq table_types; struct table_tailq tables; + struct regarray_tailq regarrays; struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; struct rte_swx_table_state *table_state; + struct regarray_runtime *regarray_runtime; struct instruction *instructions; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; @@ -998,6 +1131,7 @@ struct rte_swx_pipeline { uint32_t n_extern_funcs; uint32_t n_actions; uint32_t n_tables; + uint32_t n_regarrays; uint32_t n_headers; uint32_t thread_id; uint32_t port_id; @@ -4622,222 +4756,1124 @@ instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p) } /* - * jmp. + * Register array. */ -static struct action * -action_find(struct rte_swx_pipeline *p, const char *name); - -static int -instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(n_tokens == 2, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - instr->type = INSTR_JMP; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name); static int -instr_jmp_valid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regprefetch_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *regarray = tokens[1], *idx = tokens[2]; + struct regarray *r; + struct field *fidx; + uint32_t idx_struct_id, idx_val; CHECK(n_tokens == 3, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); + + /* REGPREFETCH_RH, REGPREFETCH_RM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGPREFETCH_RM; + if (idx[0] == 'h') + instr->type = INSTR_REGPREFETCH_RH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = 0; /* Unused. */ + return 0; + } - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + /* REGPREFETCH_RI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - instr->type = INSTR_JMP_VALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; + instr->type = INSTR_REGPREFETCH_RI; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = 0; /* Unused. */ return 0; } static int -instr_jmp_invalid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regrd_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3]; + struct regarray *r; + struct field *fdst, *fidx; + uint32_t dst_struct_id, idx_struct_id, idx_val; - CHECK(n_tokens == 3, EINVAL); + CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); - instr->type = INSTR_JMP_INVALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; - return 0; -} + /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGRD_MRM; + if (dst[0] == 'h' && idx[0] != 'h') + instr->type = INSTR_REGRD_HRM; + if (dst[0] != 'h' && idx[0] == 'h') + instr->type = INSTR_REGRD_MRH; + if (dst[0] == 'h' && idx[0] == 'h') + instr->type = INSTR_REGRD_HRH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; + return 0; + } -static int -instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + /* REGRD_MRI, REGRD_HRI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGRD_MRI; + if (dst[0] == 'h') + instr->type = INSTR_REGRD_HRI; - instr->type = INSTR_JMP_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; return 0; } static int -instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regwr_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; - strcpy(data->jmp_label, tokens[1]); + CHECK(n_tokens == 4, EINVAL); - instr->type = INSTR_JMP_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + r = regarray_find(p, regarray); + CHECK(r, EINVAL); -static int -instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGWR_RMM; + if (idx[0] == 'h' && src[0] != 'h') + instr->type = INSTR_REGWR_RHM; + if (idx[0] != 'h' && src[0] == 'h') + instr->type = INSTR_REGWR_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGWR_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + /* REGWR_RHI, REGWR_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGWR_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGWR_RHI; - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; + return 0; + } - instr->type = INSTR_JMP_ACTION_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + /* REGWR_RIH, REGWR_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); -static int -instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + instr->type = INSTR_REGWR_RIM; + if (src[0] == 'h') + instr->type = INSTR_REGWR_RIH; - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - strcpy(data->jmp_label, tokens[1]); + /* REGWR_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGWR_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; - instr->type = INSTR_JMP_ACTION_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; return 0; } static int -instr_jmp_eq_translate(struct rte_swx_pipeline *p, +instr_regadd_translate(struct rte_swx_pipeline *p, struct action *action, char **tokens, int n_tokens, struct instruction *instr, - struct instruction_data *data) + struct instruction_data *data __rte_unused) { - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGADD_RMM; + if (idx[0] == 'h' && src[0] != 'h') + instr->type = INSTR_REGADD_RHM; + if (idx[0] != 'h' && src[0] == 'h') + instr->type = INSTR_REGADD_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGADD_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - /* JMP_EQ or JMP_EQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_EQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_EQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + /* REGADD_RHI, REGADD_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; + instr->type = INSTR_REGADD_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGADD_RHI; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; return 0; } - /* JMP_EQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* REGADD_RIH, REGADD_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - if (a[0] == 'h') - b_val = hton64(b_val) >> (64 - fa->n_bits); + instr->type = INSTR_REGADD_RIM; + if (src[0] == 'h') + instr->type = INSTR_REGADD_RIH; - instr->type = INSTR_JMP_EQ_I; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b_val = b_val; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } + + /* REGADD_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); + + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGADD_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; return 0; } -static int -instr_jmp_neq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +static inline uint64_t * +instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip) { - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + return r->regarray; +} + +static inline uint64_t +instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return idx; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; + + return idx; +} + +#else + +#define instr_regarray_idx_nbo instr_regarray_idx_hbo + +#endif + +static inline uint64_t +instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint64_t idx = ip->regarray.idx_val & r->size_mask; + + return idx; +} + +static inline uint64_t +instr_regarray_src_hbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + uint64_t src = src64 & src64_mask; + + return src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_src_nbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); + + return src; +} + +#else + +#define instr_regarray_src_nbo instr_regarray_src_hbo + +#endif + +static inline void +instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); + +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline void +instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +#else + +#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set + +#endif + +static inline void +instr_regprefetch_rh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + rte_prefetch0(®array[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_rm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + rte_prefetch0(®array[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_ri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + rte_prefetch0(®array[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; + + /* Thread. */ + thread_ip_inc(p); +} + +/* + * jmp. + */ +static struct action * +action_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_valid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_VALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_invalid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_INVALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_eq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_EQ or JMP_EQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_EQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_EQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_EQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = hton64(b_val) >> (64 - fa->n_bits); + + instr->type = INSTR_JMP_EQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = b_val; + return 0; +} + +static int +instr_jmp_neq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; uint64_t b_val; uint32_t a_struct_id, b_struct_id; @@ -5475,6 +6511,38 @@ instr_translate(struct rte_swx_pipeline *p, instr, data); + if (!strcmp(tokens[tpos], "regprefetch")) + return instr_regprefetch_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regrd")) + return instr_regrd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regwr")) + return instr_regwr_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regadd")) + return instr_regadd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + if (!strcmp(tokens[tpos], "table")) return instr_table_translate(p, action, @@ -6111,6 +7179,37 @@ static instr_exec_t instruction_table[] = { [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec, [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec, + [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec, + [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec, + [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec, + + [INSTR_REGRD_HRH] = instr_regrd_hrh_exec, + [INSTR_REGRD_HRM] = instr_regrd_hrm_exec, + [INSTR_REGRD_MRH] = instr_regrd_mrh_exec, + [INSTR_REGRD_MRM] = instr_regrd_mrm_exec, + [INSTR_REGRD_HRI] = instr_regrd_hri_exec, + [INSTR_REGRD_MRI] = instr_regrd_mri_exec, + + [INSTR_REGWR_RHH] = instr_regwr_rhh_exec, + [INSTR_REGWR_RHM] = instr_regwr_rhm_exec, + [INSTR_REGWR_RMH] = instr_regwr_rmh_exec, + [INSTR_REGWR_RMM] = instr_regwr_rmm_exec, + [INSTR_REGWR_RHI] = instr_regwr_rhi_exec, + [INSTR_REGWR_RMI] = instr_regwr_rmi_exec, + [INSTR_REGWR_RIH] = instr_regwr_rih_exec, + [INSTR_REGWR_RIM] = instr_regwr_rim_exec, + [INSTR_REGWR_RII] = instr_regwr_rii_exec, + + [INSTR_REGADD_RHH] = instr_regadd_rhh_exec, + [INSTR_REGADD_RHM] = instr_regadd_rhm_exec, + [INSTR_REGADD_RMH] = instr_regadd_rmh_exec, + [INSTR_REGADD_RMM] = instr_regadd_rmm_exec, + [INSTR_REGADD_RHI] = instr_regadd_rhi_exec, + [INSTR_REGADD_RMI] = instr_regadd_rmi_exec, + [INSTR_REGADD_RIH] = instr_regadd_rih_exec, + [INSTR_REGADD_RIM] = instr_regadd_rim_exec, + [INSTR_REGADD_RII] = instr_regadd_rii_exec, + [INSTR_TABLE] = instr_table_exec, [INSTR_EXTERN_OBJ] = instr_extern_obj_exec, [INSTR_EXTERN_FUNC] = instr_extern_func_exec, @@ -6823,6 +7922,132 @@ table_free(struct rte_swx_pipeline *p) } } +/* + * Register array. + */ +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name) +{ + struct regarray *elem; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct regarray * +regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id) +{ + struct regarray *elem = NULL; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (elem->id == id) + return elem; + + return NULL; +} + +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size, + uint64_t init_val) +{ + struct regarray *r; + + CHECK(p, EINVAL); + + CHECK_NAME(name, EINVAL); + CHECK(!regarray_find(p, name), EEXIST); + + CHECK(size, EINVAL); + size = rte_align32pow2(size); + + /* Memory allocation. */ + r = calloc(1, sizeof(struct regarray)); + CHECK(r, ENOMEM); + + /* Node initialization. */ + strcpy(r->name, name); + r->init_val = init_val; + r->size = size; + r->id = p->n_regarrays; + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->regarrays, r, node); + p->n_regarrays++; + + return 0; +} + +static int +regarray_build(struct rte_swx_pipeline *p) +{ + struct regarray *regarray; + + if (!p->n_regarrays) + return 0; + + p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime)); + CHECK(p->regarray_runtime, ENOMEM); + + TAILQ_FOREACH(regarray, &p->regarrays, node) { + struct regarray_runtime *r = &p->regarray_runtime[regarray->id]; + uint32_t i; + + r->regarray = env_malloc(regarray->size * sizeof(uint64_t), + RTE_CACHE_LINE_SIZE, + p->numa_node); + CHECK(r->regarray, ENOMEM); + + if (regarray->init_val) + for (i = 0; i < regarray->size; i++) + r->regarray[i] = regarray->init_val; + + r->size_mask = regarray->size - 1; + } + + return 0; +} + +static void +regarray_build_free(struct rte_swx_pipeline *p) +{ + uint32_t i; + + if (!p->regarray_runtime) + return; + + for (i = 0; i < p->n_regarrays; i++) { + struct regarray *regarray = regarray_find_by_id(p, i); + struct regarray_runtime *r = &p->regarray_runtime[i]; + + env_free(r->regarray, regarray->size * sizeof(uint64_t)); + } + + free(p->regarray_runtime); + p->regarray_runtime = NULL; +} + +static void +regarray_free(struct rte_swx_pipeline *p) +{ + regarray_build_free(p); + + for ( ; ; ) { + struct regarray *elem; + + elem = TAILQ_FIRST(&p->regarrays); + if (!elem) + break; + + TAILQ_REMOVE(&p->regarrays, elem, node); + free(elem); + } +} + /* * Pipeline. */ @@ -6851,6 +8076,7 @@ rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node) TAILQ_INIT(&pipeline->actions); TAILQ_INIT(&pipeline->table_types); TAILQ_INIT(&pipeline->tables); + TAILQ_INIT(&pipeline->regarrays); pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */ pipeline->numa_node = numa_node; @@ -6867,6 +8093,7 @@ rte_swx_pipeline_free(struct rte_swx_pipeline *p) free(p->instructions); + regarray_free(p); table_state_free(p); table_free(p); action_free(p); @@ -6951,10 +8178,15 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p) if (status) goto error; + status = regarray_build(p); + if (status) + goto error; + p->build_done = 1; return 0; error: + regarray_build_free(p); table_state_build_free(p); table_build_free(p); action_build_free(p); @@ -7015,6 +8247,7 @@ rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p, pipeline->n_ports_out = p->n_ports_out; pipeline->n_actions = n_actions; pipeline->n_tables = n_tables; + pipeline->n_regarrays = p->n_regarrays; return 0; } @@ -7222,3 +8455,64 @@ rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p, port->type->ops.stats_read(port->obj, stats); return 0; } + +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray) +{ + struct regarray *r; + + if (!p || !regarray) + return -EINVAL; + + r = regarray_find_by_id(p, regarray_id); + if (!r) + return -EINVAL; + + strcpy(regarray->name, r->name); + regarray->size = r->size; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name || !value) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + *value = r->regarray[regarray_index]; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + r->regarray[regarray_index] = value; + return 0; +} diff --git a/lib/librte_pipeline/rte_swx_pipeline.h b/lib/librte_pipeline/rte_swx_pipeline.h index f0a2cef77..1c9b2eb12 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.h +++ b/lib/librte_pipeline/rte_swx_pipeline.h @@ -616,6 +616,30 @@ rte_swx_pipeline_table_config(struct rte_swx_pipeline *p, const char *args, uint32_t size); +/** + * Pipeline register array configure + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Register array name. + * @param[in] size + * Number of registers in the array. Each register is 64-bit in size. + * @param[in] init_val + * Initial value for every register in the array. The recommended value is 0. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Register array with this name already exists. + */ +__rte_experimental +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size, + uint64_t init_val); + /** * Pipeline instructions configure * diff --git a/lib/librte_pipeline/rte_swx_pipeline_spec.c b/lib/librte_pipeline/rte_swx_pipeline_spec.c index f7884491b..0d0bca2ed 100644 --- a/lib/librte_pipeline/rte_swx_pipeline_spec.c +++ b/lib/librte_pipeline/rte_swx_pipeline_spec.c @@ -940,6 +940,81 @@ table_block_parse(struct table_spec *s, return -EINVAL; } +/* + * regarray. + * + * regarray NAME size SIZE initval INITVAL + */ +struct regarray_spec { + char *name; + uint64_t init_val; + uint32_t size; +}; + +static void +regarray_spec_free(struct regarray_spec *s) +{ + if (!s) + return; + + free(s->name); + s->name = NULL; +} + +static int +regarray_statement_parse(struct regarray_spec *s, + char **tokens, + uint32_t n_tokens, + uint32_t n_lines, + uint32_t *err_line, + const char **err_msg) +{ + char *p; + + /* Check format. */ + if ((n_tokens != 6) || + strcmp(tokens[2], "size") || + strcmp(tokens[4], "initval")) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid regarray statement."; + return -EINVAL; + } + + /* spec. */ + s->name = strdup(tokens[1]); + if (!s->name) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Memory allocation failed."; + return -ENOMEM; + } + + p = tokens[3]; + s->size = strtoul(p, &p, 0); + if (p[0] || !s->size) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid size argument."; + return -EINVAL; + } + + p = tokens[5]; + s->init_val = strtoull(p, &p, 0); + if (p[0]) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid initval argument."; + return -EINVAL; + } + + return 0; +} + /* * apply. * @@ -1066,6 +1141,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, struct metadata_spec metadata_spec = {0}; struct action_spec action_spec = {0}; struct table_spec table_spec = {0}; + struct regarray_spec regarray_spec = {0}; struct apply_spec apply_spec = {0}; uint32_t n_lines; uint32_t block_mask = 0; @@ -1405,6 +1481,34 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, continue; } + /* regarray. */ + if (!strcmp(tokens[0], "regarray")) { + status = regarray_statement_parse(®array_spec, + tokens, + n_tokens, + n_lines, + err_line, + err_msg); + if (status) + goto error; + + status = rte_swx_pipeline_regarray_config(p, + regarray_spec.name, + regarray_spec.size, + regarray_spec.init_val); + if (status) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Register array configuration error."; + goto error; + } + + regarray_spec_free(®array_spec); + + continue; + } + /* apply. */ if (!strcmp(tokens[0], "apply")) { status = apply_statement_parse(&block_mask, @@ -1457,6 +1561,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, metadata_spec_free(&metadata_spec); action_spec_free(&action_spec); table_spec_free(&table_spec); + regarray_spec_free(®array_spec); apply_spec_free(&apply_spec); return status; } diff --git a/lib/librte_pipeline/version.map b/lib/librte_pipeline/version.map index 87c826f1b..26a9edf47 100644 --- a/lib/librte_pipeline/version.map +++ b/lib/librte_pipeline/version.map @@ -102,4 +102,10 @@ EXPERIMENTAL { rte_swx_pipeline_table_state_get; rte_swx_pipeline_table_state_set; rte_swx_pipeline_table_type_register; + + #added in 21.05 + rte_swx_ctl_pipeline_regarray_read; + rte_swx_ctl_pipeline_regarray_write; + rte_swx_ctl_regarray_info_get; + rte_swx_pipeline_regarray_config; }; -- 2.17.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* [dpdk-dev] [PATCH v5 2/2] pipeline: add meter support to the SWX pipeline 2021-03-16 11:40 ` [dpdk-dev] [PATCH v5 1/2] pipeline: add register array support to " Cristian Dumitrescu @ 2021-03-16 11:40 ` Cristian Dumitrescu 2021-03-24 18:19 ` Thomas Monjalon 0 siblings, 1 reply; 8+ messages in thread From: Cristian Dumitrescu @ 2021-03-16 11:40 UTC (permalink / raw) To: dev Meter arrays are stateful objects that are updated by the data plane and configured & monitored by the control plane. The meters implement the RFC 2698 Two Rate Three Color Marker (trTCM) algorithm. Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com> --- examples/pipeline/cli.c | 471 ++++++ examples/pipeline/examples/meter.cli | 31 + examples/pipeline/examples/meter.spec | 63 + lib/librte_pipeline/rte_swx_ctl.h | 155 ++ lib/librte_pipeline/rte_swx_pipeline.c | 1684 ++++++++++++++++--- lib/librte_pipeline/rte_swx_pipeline.h | 22 + lib/librte_pipeline/rte_swx_pipeline_spec.c | 91 + lib/librte_pipeline/version.map | 7 + 8 files changed, 2328 insertions(+), 196 deletions(-) create mode 100644 examples/pipeline/examples/meter.cli create mode 100644 examples/pipeline/examples/meter.spec diff --git a/examples/pipeline/cli.c b/examples/pipeline/cli.c index cdf8f13f3..718dd07d4 100644 --- a/examples/pipeline/cli.c +++ b/examples/pipeline/cli.c @@ -1108,6 +1108,398 @@ cmd_pipeline_regwr(char **tokens, } } +static const char cmd_pipeline_meter_profile_add_help[] = +"pipeline <pipeline_name> meter profile <profile_name> add " + "cir <cir> pir <pir> cbs <cbs> pbs <pbs>\n"; + +static void +cmd_pipeline_meter_profile_add(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct rte_meter_trtcm_params params; + struct pipeline *p; + const char *profile_name; + int status; + + if (n_tokens != 14) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + if (strcmp(tokens[3], "profile")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile"); + return; + } + + profile_name = tokens[4]; + + if (strcmp(tokens[5], "add")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "add"); + return; + } + + if (strcmp(tokens[6], "cir")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cir"); + return; + } + + if (parser_read_uint64(¶ms.cir, tokens[7])) { + snprintf(out, out_size, MSG_ARG_INVALID, "cir"); + return; + } + + if (strcmp(tokens[8], "pir")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pir"); + return; + } + + if (parser_read_uint64(¶ms.pir, tokens[9])) { + snprintf(out, out_size, MSG_ARG_INVALID, "pir"); + return; + } + + if (strcmp(tokens[10], "cbs")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cbs"); + return; + } + + if (parser_read_uint64(¶ms.cbs, tokens[11])) { + snprintf(out, out_size, MSG_ARG_INVALID, "cbs"); + return; + } + + if (strcmp(tokens[12], "pbs")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pbs"); + return; + } + + if (parser_read_uint64(¶ms.pbs, tokens[13])) { + snprintf(out, out_size, MSG_ARG_INVALID, "pbs"); + return; + } + + status = rte_swx_ctl_meter_profile_add(p->p, profile_name, ¶ms); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + +static const char cmd_pipeline_meter_profile_delete_help[] = +"pipeline <pipeline_name> meter profile <profile_name> delete\n"; + +static void +cmd_pipeline_meter_profile_delete(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *profile_name; + int status; + + if (n_tokens != 6) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + if (strcmp(tokens[3], "profile")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile"); + return; + } + + profile_name = tokens[4]; + + if (strcmp(tokens[5], "delete")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "delete"); + return; + } + + status = rte_swx_ctl_meter_profile_delete(p->p, profile_name); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + +static const char cmd_pipeline_meter_reset_help[] = +"pipeline <pipeline_name> meter <meter_array_name> from <index0> to <index1> " + "reset\n"; + +static void +cmd_pipeline_meter_reset(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint32_t idx0, idx1; + + if (n_tokens != 9) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + name = tokens[3]; + + if (strcmp(tokens[4], "from")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "from"); + return; + } + + if (parser_read_uint32(&idx0, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index0"); + return; + } + + if (strcmp(tokens[6], "to")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "to"); + return; + } + + if (parser_read_uint32(&idx1, tokens[7]) || (idx1 < idx0)) { + snprintf(out, out_size, MSG_ARG_INVALID, "index1"); + return; + } + + if (strcmp(tokens[8], "reset")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "reset"); + return; + } + + for ( ; idx0 <= idx1; idx0++) { + int status; + + status = rte_swx_ctl_meter_reset(p->p, name, idx0); + if (status) { + snprintf(out, out_size, "Command failed for index %u.\n", idx0); + return; + } + } +} + +static const char cmd_pipeline_meter_set_help[] = +"pipeline <pipeline_name> meter <meter_array_name> from <index0> to <index1> " + "set profile <profile_name>\n"; + +static void +cmd_pipeline_meter_set(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name, *profile_name; + uint32_t idx0, idx1; + + if (n_tokens != 11) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + name = tokens[3]; + + if (strcmp(tokens[4], "from")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "from"); + return; + } + + if (parser_read_uint32(&idx0, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index0"); + return; + } + + if (strcmp(tokens[6], "to")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "to"); + return; + } + + if (parser_read_uint32(&idx1, tokens[7]) || (idx1 < idx0)) { + snprintf(out, out_size, MSG_ARG_INVALID, "index1"); + return; + } + + if (strcmp(tokens[8], "set")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "set"); + return; + } + + if (strcmp(tokens[9], "profile")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile"); + return; + } + + profile_name = tokens[10]; + + for ( ; idx0 <= idx1; idx0++) { + int status; + + status = rte_swx_ctl_meter_set(p->p, name, idx0, profile_name); + if (status) { + snprintf(out, out_size, "Command failed for index %u.\n", idx0); + return; + } + } +} + +static const char cmd_pipeline_meter_stats_help[] = +"pipeline <pipeline_name> meter <meter_array_name> from <index0> to <index1> " + "stats\n"; + +static void +cmd_pipeline_meter_stats(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct rte_swx_ctl_meter_stats stats; + struct pipeline *p; + const char *name; + uint32_t idx0, idx1; + + if (n_tokens != 9) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "meter")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter"); + return; + } + + name = tokens[3]; + + if (strcmp(tokens[4], "from")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "from"); + return; + } + + if (parser_read_uint32(&idx0, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index0"); + return; + } + + if (strcmp(tokens[6], "to")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "to"); + return; + } + + if (parser_read_uint32(&idx1, tokens[7]) || (idx1 < idx0)) { + snprintf(out, out_size, MSG_ARG_INVALID, "index1"); + return; + } + + if (strcmp(tokens[8], "stats")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "stats"); + return; + } + + /* Table header. */ + snprintf(out, out_size, "+-%7s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+\n", + "-------", + "----------------", "----------------", "----------------", + "----------------", "----------------", "----------------"); + out_size -= strlen(out); + out += strlen(out); + + snprintf(out, out_size, "| %4s | %16s | %16s | %16s | %16s | %16s | %16s |\n", + "METER #", + "GREEN (packets)", "YELLOW (packets)", "RED (packets)", + "GREEN (bytes)", "YELLOW (bytes)", "RED (bytes)"); + out_size -= strlen(out); + out += strlen(out); + + snprintf(out, out_size, "+-%7s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+-%16s-+\n", + "-------", + "----------------", "----------------", "----------------", + "----------------", "----------------", "----------------"); + out_size -= strlen(out); + out += strlen(out); + + /* Table rows. */ + for ( ; idx0 <= idx1; idx0++) { + int status; + + status = rte_swx_ctl_meter_stats_read(p->p, name, idx0, &stats); + if (status) { + snprintf(out, out_size, "Pipeline meter stats error at index %u.\n", idx0); + out_size -= strlen(out); + out += strlen(out); + return; + } + + snprintf(out, out_size, "| %7d | %16" PRIx64 " | %16" PRIx64 " | %16" PRIx64 + " | %16" PRIx64 " | %16" PRIx64 " | %16" PRIx64 " |\n", + idx0, + stats.n_pkts[RTE_COLOR_GREEN], + stats.n_pkts[RTE_COLOR_YELLOW], + stats.n_pkts[RTE_COLOR_RED], + stats.n_bytes[RTE_COLOR_GREEN], + stats.n_bytes[RTE_COLOR_YELLOW], + stats.n_bytes[RTE_COLOR_RED]); + out_size -= strlen(out); + out += strlen(out); + } +} + static const char cmd_pipeline_stats_help[] = "pipeline <pipeline_name> stats\n"; @@ -1303,6 +1695,11 @@ cmd_help(char **tokens, "\tpipeline table update\n" "\tpipeline regrd\n" "\tpipeline regwr\n" + "\tpipeline meter profile add\n" + "\tpipeline meter profile delete\n" + "\tpipeline meter reset\n" + "\tpipeline meter set\n" + "\tpipeline meter stats\n" "\tpipeline stats\n" "\tthread pipeline enable\n" "\tthread pipeline disable\n\n"); @@ -1367,6 +1764,43 @@ cmd_help(char **tokens, return; } + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 4) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "profile") + && !strcmp(tokens[3], "add")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_profile_add_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 4) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "profile") + && !strcmp(tokens[3], "delete")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_profile_delete_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 3) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "reset")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_reset_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 3) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "set")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_set_help); + return; + } + + if (!strcmp(tokens[0], "pipeline") && + (n_tokens == 3) && !strcmp(tokens[1], "meter") + && !strcmp(tokens[2], "stats")) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_meter_stats_help); + return; + } + if ((strcmp(tokens[0], "pipeline") == 0) && (n_tokens == 2) && (strcmp(tokens[1], "stats") == 0)) { snprintf(out, out_size, "\n%s\n", cmd_pipeline_stats_help); @@ -1481,6 +1915,43 @@ cli_process(char *in, char *out, size_t out_size, void *obj) return; } + if ((n_tokens >= 6) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[3], "profile") == 0) && + (strcmp(tokens[5], "add") == 0)) { + cmd_pipeline_meter_profile_add(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 6) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[3], "profile") == 0) && + (strcmp(tokens[5], "delete") == 0)) { + cmd_pipeline_meter_profile_delete(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 9) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[8], "reset") == 0)) { + cmd_pipeline_meter_reset(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 9) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[8], "set") == 0)) { + cmd_pipeline_meter_set(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 9) && + (strcmp(tokens[2], "meter") == 0) && + (strcmp(tokens[8], "stats") == 0)) { + cmd_pipeline_meter_stats(tokens, n_tokens, out, out_size, obj); + return; + } + if ((n_tokens >= 3) && (strcmp(tokens[2], "stats") == 0)) { cmd_pipeline_stats(tokens, n_tokens, out, out_size, diff --git a/examples/pipeline/examples/meter.cli b/examples/pipeline/examples/meter.cli new file mode 100644 index 000000000..b29ed2402 --- /dev/null +++ b/examples/pipeline/examples/meter.cli @@ -0,0 +1,31 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2020 Intel Corporation + +; Example command line: +; ./build/examples/dpdk-pipeline -l0-1 -- -s ./examples/pipeline/examples/meter.cli + +mempool MEMPOOL0 buffer 2304 pool 32K cache 256 cpu 0 + +link LINK0 dev 0000:18:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK1 dev 0000:18:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK2 dev 0000:3b:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on +link LINK3 dev 0000:3b:00.1 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on + +pipeline PIPELINE0 create 0 + +pipeline PIPELINE0 port in 0 link LINK0 rxq 0 bsz 32 +pipeline PIPELINE0 port in 1 link LINK1 rxq 0 bsz 32 +pipeline PIPELINE0 port in 2 link LINK2 rxq 0 bsz 32 +pipeline PIPELINE0 port in 3 link LINK3 rxq 0 bsz 32 + +pipeline PIPELINE0 port out 0 link LINK0 txq 0 bsz 32 +pipeline PIPELINE0 port out 1 link LINK1 txq 0 bsz 32 +pipeline PIPELINE0 port out 2 link LINK2 txq 0 bsz 32 +pipeline PIPELINE0 port out 3 link LINK3 txq 0 bsz 32 + +pipeline PIPELINE0 build ./examples/pipeline/examples/meter.spec + +pipeline PIPELINE0 meter profile platinum add cir 46000000 pir 138000000 cbs 1000000 pbs 1000000 +pipeline PIPELINE0 meter meters from 0 to 15 set profile platinum + +thread 1 pipeline PIPELINE0 enable diff --git a/examples/pipeline/examples/meter.spec b/examples/pipeline/examples/meter.spec new file mode 100644 index 000000000..43b8170d7 --- /dev/null +++ b/examples/pipeline/examples/meter.spec @@ -0,0 +1,63 @@ +; SPDX-License-Identifier: BSD-3-Clause +; Copyright(c) 2021 Intel Corporation + +; This program is setting up an array of Two Rate Three Color Marker (trTCM) meters called "meters". +; Every input packet (Ethernet/IPv4) is metered by the meter at the location indexed by the IPv4 +; header "Source Address" field. All green packets are sent out on port 0, the yellow ones on port 1 +; and the red ones on port 2. +; +; The "meter stats" CLI command can be used to read the packet and byte statistics counters of any +; meter in the array. + +// +// Headers. +// +struct ethernet_h { + bit<48> dst_addr + bit<48> src_addr + bit<16> ethertype +} + +struct ipv4_h { + bit<8> ver_ihl + bit<8> diffserv + bit<16> total_len + bit<16> identification + bit<16> flags_offset + bit<8> ttl + bit<8> protocol + bit<16> hdr_checksum + bit<32> src_addr + bit<32> dst_addr +} + +header ethernet instanceof ethernet_h +header ipv4 instanceof ipv4_h + +// +// Meta-data. +// +struct metadata_t { + bit<32> port_in + bit<32> port_out +} + +metadata instanceof metadata_t + +// +// Registers. +// +metarray meters size 65536 + +// +// Pipeline. +// +apply { + rx m.port_in + extract h.ethernet + extract h.ipv4 + meter meters h.ipv4.src_addr h.ipv4.total_len 0 m.port_out + emit h.ethernet + emit h.ipv4 + tx m.port_out +} diff --git a/lib/librte_pipeline/rte_swx_ctl.h b/lib/librte_pipeline/rte_swx_ctl.h index c0a55c6f0..b8f22c516 100644 --- a/lib/librte_pipeline/rte_swx_ctl.h +++ b/lib/librte_pipeline/rte_swx_ctl.h @@ -18,6 +18,7 @@ extern "C" { #include <stdio.h> #include <rte_compat.h> +#include <rte_meter.h> #include "rte_swx_port.h" #include "rte_swx_table.h" @@ -49,6 +50,9 @@ struct rte_swx_ctl_pipeline_info { /** Number of register arrays. */ uint32_t n_regarrays; + + /** Number of meter arrays. */ + uint32_t n_metarrays; }; /** @@ -636,6 +640,157 @@ rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, uint32_t regarray_index, uint64_t value); +/* + * Meter Array Query and Configuration API. + */ + +/** Meter array info. */ +struct rte_swx_ctl_metarray_info { + /** Meter array name. */ + char name[RTE_SWX_CTL_NAME_SIZE]; + + /** Meter array size. */ + uint32_t size; +}; + +/** + * Meter array info get + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_id + * Meter array ID (0 .. *n_metarrays* - 1). + * @param[out] metarray + * Meter array info. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p, + uint32_t metarray_id, + struct rte_swx_ctl_metarray_info *metarray); + +/** + * Meter profile add + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Meter profile name. + * @param[in] params + * Meter profile parameters. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Meter profile with this name already exists. + */ +__rte_experimental +int +rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p, + const char *name, + struct rte_meter_trtcm_params *params); + +/** + * Meter profile delete + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Meter profile name. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -EBUSY: Meter profile is currently in use. + */ +__rte_experimental +int +rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p, + const char *name); + +/** + * Meter reset + * + * Reset a meter within a given meter array to use the default profile that + * causes all the input packets to be colored as green. It is the responsibility + * of the control plane to make sure this meter is not used by the data plane + * pipeline before calling this function. + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_name + * Meter array name. + * @param[in] metarray_index + * Meter index within the meter array. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index); + +/** + * Meter set + * + * Set a meter within a given meter array to use a specific profile. It is the + * responsibility of the control plane to make sure this meter is not used by + * the data plane pipeline before calling this function. + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_name + * Meter array name. + * @param[in] metarray_index + * Meter index within the meter array. + * @param[in] profile_name + * Existing meter profile name. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_meter_set(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + const char *profile_name); + +/** Meter statistics counters. */ +struct rte_swx_ctl_meter_stats { + /** Number of packets tagged by the meter for each color. */ + uint64_t n_pkts[RTE_COLORS]; + + /** Number of bytes tagged by the meter for each color. */ + uint64_t n_bytes[RTE_COLORS]; +}; + +/** + * Meter statistics counters read + * + * @param[in] p + * Pipeline handle. + * @param[in] metarray_name + * Meter array name. + * @param[in] metarray_index + * Meter index within the meter array. + * @param[out] stats + * Meter statistics counters. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + struct rte_swx_ctl_meter_stats *stats); + /** * Pipeline control free * diff --git a/lib/librte_pipeline/rte_swx_pipeline.c b/lib/librte_pipeline/rte_swx_pipeline.c index 5be87ca4f..5257a5360 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.c +++ b/lib/librte_pipeline/rte_swx_pipeline.c @@ -12,6 +12,8 @@ #include <rte_common.h> #include <rte_prefetch.h> #include <rte_byteorder.h> +#include <rte_cycles.h> +#include <rte_meter.h> #include "rte_swx_pipeline.h" #include "rte_swx_ctl.h" @@ -458,6 +460,31 @@ enum instruction_type { INSTR_REGADD_RIM, /* index = I, src = MEFT */ INSTR_REGADD_RII, /* index = I, src = I */ + /* metprefetch METARRAY index + * prefetch METARRAY[index] + * index = HMEFTI + */ + INSTR_METPREFETCH_H, /* index = H */ + INSTR_METPREFETCH_M, /* index = MEFT */ + INSTR_METPREFETCH_I, /* index = I */ + + /* meter METARRAY index length color_in color_out + * color_out = meter(METARRAY[index], length, color_in) + * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF + */ + INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */ + INSTR_METER_HHI, /* index = H, length = H, color_in = I */ + INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */ + INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */ + INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */ + INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */ + INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */ + INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */ + INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */ + INSTR_METER_IHI, /* index = I, length = H, color_in = I */ + INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */ + INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */ + /* table TABLE */ INSTR_TABLE, @@ -607,6 +634,25 @@ struct instr_regarray { }; }; +struct instr_meter { + uint8_t metarray_id; + uint8_t pad[3]; + + union { + struct instr_operand idx; + uint32_t idx_val; + }; + + struct instr_operand length; + + union { + struct instr_operand color_in; + uint32_t color_in_val; + }; + + struct instr_operand color_out; +}; + struct instr_dma { struct { uint8_t header_id[8]; @@ -642,6 +688,7 @@ struct instruction { struct instr_hdr_validity valid; struct instr_dst_src mov; struct instr_regarray regarray; + struct instr_meter meter; struct instr_dma dma; struct instr_dst_src alu; struct instr_table table; @@ -739,6 +786,43 @@ struct regarray_runtime { uint32_t size_mask; }; +/* + * Meter array. + */ +struct meter_profile { + TAILQ_ENTRY(meter_profile) node; + char name[RTE_SWX_NAME_SIZE]; + struct rte_meter_trtcm_params params; + struct rte_meter_trtcm_profile profile; + uint32_t n_users; +}; + +TAILQ_HEAD(meter_profile_tailq, meter_profile); + +struct metarray { + TAILQ_ENTRY(metarray) node; + char name[RTE_SWX_NAME_SIZE]; + uint32_t size; + uint32_t id; +}; + +TAILQ_HEAD(metarray_tailq, metarray); + +struct meter { + struct rte_meter_trtcm m; + struct meter_profile *profile; + enum rte_color color_mask; + uint8_t pad[20]; + + uint64_t n_pkts[RTE_COLORS]; + uint64_t n_bytes[RTE_COLORS]; +}; + +struct metarray_runtime { + struct meter *metarray; + uint32_t size_mask; +}; + /* * Pipeline. */ @@ -1115,12 +1199,15 @@ struct rte_swx_pipeline { struct table_type_tailq table_types; struct table_tailq tables; struct regarray_tailq regarrays; + struct meter_profile_tailq meter_profiles; + struct metarray_tailq metarrays; struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; struct rte_swx_table_state *table_state; struct regarray_runtime *regarray_runtime; + struct metarray_runtime *metarray_runtime; struct instruction *instructions; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; @@ -1132,6 +1219,7 @@ struct rte_swx_pipeline { uint32_t n_actions; uint32_t n_tables; uint32_t n_regarrays; + uint32_t n_metarrays; uint32_t n_headers; uint32_t thread_id; uint32_t port_id; @@ -5658,255 +5746,1078 @@ instr_regadd_rii_exec(struct rte_swx_pipeline *p) } /* - * jmp. + * metarray. */ -static struct action * -action_find(struct rte_swx_pipeline *p, const char *name); - -static int -instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(n_tokens == 2, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - instr->type = INSTR_JMP; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} - -static int -instr_jmp_valid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct header *h; - - CHECK(n_tokens == 3, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); - - instr->type = INSTR_JMP_VALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; - return 0; -} +static struct metarray * +metarray_find(struct rte_swx_pipeline *p, const char *name); static int -instr_jmp_invalid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, +instr_metprefetch_translate(struct rte_swx_pipeline *p, + struct action *action, char **tokens, int n_tokens, struct instruction *instr, - struct instruction_data *data) + struct instruction_data *data __rte_unused) { - struct header *h; + char *metarray = tokens[1], *idx = tokens[2]; + struct metarray *m; + struct field *fidx; + uint32_t idx_struct_id, idx_val; CHECK(n_tokens == 3, EINVAL); - strcpy(data->jmp_label, tokens[1]); + m = metarray_find(p, metarray); + CHECK(m, EINVAL); - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + /* METPREFETCH_H, METPREFETCH_M. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_METPREFETCH_M; + if (idx[0] == 'h') + instr->type = INSTR_METPREFETCH_H; - instr->type = INSTR_JMP_INVALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; + instr->meter.metarray_id = m->id; + instr->meter.idx.struct_id = (uint8_t)idx_struct_id; + instr->meter.idx.n_bits = fidx->n_bits; + instr->meter.idx.offset = fidx->offset / 8; + return 0; + } + + /* METPREFETCH_I. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_METPREFETCH_I; + instr->meter.metarray_id = m->id; + instr->meter.idx_val = idx_val; return 0; } static int -instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_meter_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3]; + char *color_in = tokens[4], *color_out = tokens[5]; + struct metarray *m; + struct field *fidx, *flength, *fcin, *fcout; + uint32_t idx_struct_id, length_struct_id; + uint32_t color_in_struct_id, color_out_struct_id; - strcpy(data->jmp_label, tokens[1]); + CHECK(n_tokens == 6, EINVAL); - instr->type = INSTR_JMP_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + m = metarray_find(p, metarray); + CHECK(m, EINVAL); -static int -instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + fidx = struct_field_parse(p, action, idx, &idx_struct_id); - strcpy(data->jmp_label, tokens[1]); + flength = struct_field_parse(p, action, length, &length_struct_id); + CHECK(flength, EINVAL); - instr->type = INSTR_JMP_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + fcin = struct_field_parse(p, action, color_in, &color_in_struct_id); -static int -instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id); + CHECK(fcout, EINVAL); - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */ + if (fidx && fcin) { + instr->type = INSTR_METER_MMM; + if (idx[0] == 'h' && length[0] == 'h') + instr->type = INSTR_METER_HHM; + if (idx[0] == 'h' && length[0] != 'h') + instr->type = INSTR_METER_HMM; + if (idx[0] != 'h' && length[0] == 'h') + instr->type = INSTR_METER_MHM; - strcpy(data->jmp_label, tokens[1]); + instr->meter.metarray_id = m->id; - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + instr->meter.idx.struct_id = (uint8_t)idx_struct_id; + instr->meter.idx.n_bits = fidx->n_bits; + instr->meter.idx.offset = fidx->offset / 8; - instr->type = INSTR_JMP_ACTION_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; -static int -instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id; + instr->meter.color_in.n_bits = fcin->n_bits; + instr->meter.color_in.offset = fcin->offset / 8; - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; - strcpy(data->jmp_label, tokens[1]); + return 0; + } - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */ + if (fidx && !fcin) { + uint32_t color_in_val = strtoul(color_in, &color_in, 0); + CHECK(!color_in[0], EINVAL); - instr->type = INSTR_JMP_ACTION_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + instr->type = INSTR_METER_MMI; + if (idx[0] == 'h' && length[0] == 'h') + instr->type = INSTR_METER_HHI; + if (idx[0] == 'h' && length[0] != 'h') + instr->type = INSTR_METER_HMI; + if (idx[0] != 'h' && length[0] == 'h') + instr->type = INSTR_METER_MHI; -static int -instr_jmp_eq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + instr->meter.metarray_id = m->id; - CHECK(n_tokens == 4, EINVAL); + instr->meter.idx.struct_id = (uint8_t)idx_struct_id; + instr->meter.idx.n_bits = fidx->n_bits; + instr->meter.idx.offset = fidx->offset / 8; - strcpy(data->jmp_label, tokens[1]); + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + instr->meter.color_in_val = color_in_val; - /* JMP_EQ or JMP_EQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_EQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_EQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; return 0; } - /* JMP_EQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */ + if (!fidx && fcin) { + uint32_t idx_val; - if (a[0] == 'h') - b_val = hton64(b_val) >> (64 - fa->n_bits); + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - instr->type = INSTR_JMP_EQ_I; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b_val = b_val; - return 0; -} + instr->type = INSTR_METER_IMM; + if (length[0] == 'h') + instr->type = INSTR_METER_IHM; -static int -instr_jmp_neq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + instr->meter.metarray_id = m->id; - CHECK(n_tokens == 4, EINVAL); + instr->meter.idx_val = idx_val; - strcpy(data->jmp_label, tokens[1]); + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id; + instr->meter.color_in.n_bits = fcin->n_bits; + instr->meter.color_in.offset = fcin->offset / 8; - /* JMP_NEQ or JMP_NEQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_NEQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_NEQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; return 0; } - /* JMP_NEQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* index = I, length = HMEFT, color_in = I, color_out = MEF. */ + if (!fidx && !fcin) { + uint32_t idx_val, color_in_val; - if (a[0] == 'h') + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + color_in_val = strtoul(color_in, &color_in, 0); + CHECK(!color_in[0], EINVAL); + + instr->type = INSTR_METER_IMI; + if (length[0] == 'h') + instr->type = INSTR_METER_IHI; + + instr->meter.metarray_id = m->id; + + instr->meter.idx_val = idx_val; + + instr->meter.length.struct_id = (uint8_t)length_struct_id; + instr->meter.length.n_bits = flength->n_bits; + instr->meter.length.offset = flength->offset / 8; + + instr->meter.color_in_val = color_in_val; + + instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id; + instr->meter.color_out.n_bits = fcout->n_bits; + instr->meter.color_out.offset = fcout->offset / 8; + + return 0; + } + + CHECK(0, EINVAL); +} + +static inline struct meter * +instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return &r->metarray[idx]; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline struct meter * +instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; + + return &r->metarray[idx]; +} + +#else + +#define instr_meter_idx_nbo instr_meter_idx_hbo + +#endif + +static inline struct meter * +instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint64_t idx = ip->meter.idx_val & r->size_mask; + + return &r->metarray[idx]; +} + +static inline uint32_t +instr_meter_length_hbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); + uint64_t src = src64 & src64_mask; + + return (uint32_t)src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint32_t +instr_meter_length_nbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); + + return (uint32_t)src; +} + +#else + +#define instr_meter_length_nbo instr_meter_length_hbo + +#endif + +static inline enum rte_color +instr_meter_color_in_hbo(struct thread *t, struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); + uint64_t src = src64 & src64_mask; + + return (enum rte_color)src; +} + +static inline void +instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out) +{ + uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); + + uint64_t src = (uint64_t)color_out; + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +static inline void +instr_metprefetch_h_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + + TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_metprefetch_m_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + + TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_metprefetch_i_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + + TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_hhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_hhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_hmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} +static inline void +instr_meter_hmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_mmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_ihm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_ihi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_meter_imm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imm)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} +static inline void +instr_meter_imi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imi)\n", p->thread_id); + + /* Structs. */ + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; + + /* Thread. */ + thread_ip_inc(p); +} + +/* + * jmp. + */ +static struct action * +action_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_valid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_VALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_invalid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_INVALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_eq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_EQ or JMP_EQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_EQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_EQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_EQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = hton64(b_val) >> (64 - fa->n_bits); + + instr->type = INSTR_JMP_EQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = b_val; + return 0; +} + +static int +instr_jmp_neq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_NEQ or JMP_NEQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_NEQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_NEQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_NEQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') b_val = hton64(b_val) >> (64 - fa->n_bits); instr->type = INSTR_JMP_NEQ_I; @@ -6543,6 +7454,22 @@ instr_translate(struct rte_swx_pipeline *p, instr, data); + if (!strcmp(tokens[tpos], "metprefetch")) + return instr_metprefetch_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "meter")) + return instr_meter_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + if (!strcmp(tokens[tpos], "table")) return instr_table_translate(p, action, @@ -7210,6 +8137,23 @@ static instr_exec_t instruction_table[] = { [INSTR_REGADD_RIM] = instr_regadd_rim_exec, [INSTR_REGADD_RII] = instr_regadd_rii_exec, + [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec, + [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec, + [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec, + + [INSTR_METER_HHM] = instr_meter_hhm_exec, + [INSTR_METER_HHI] = instr_meter_hhi_exec, + [INSTR_METER_HMM] = instr_meter_hmm_exec, + [INSTR_METER_HMI] = instr_meter_hmi_exec, + [INSTR_METER_MHM] = instr_meter_mhm_exec, + [INSTR_METER_MHI] = instr_meter_mhi_exec, + [INSTR_METER_MMM] = instr_meter_mmm_exec, + [INSTR_METER_MMI] = instr_meter_mmi_exec, + [INSTR_METER_IHM] = instr_meter_ihm_exec, + [INSTR_METER_IHI] = instr_meter_ihi_exec, + [INSTR_METER_IMM] = instr_meter_imm_exec, + [INSTR_METER_IMI] = instr_meter_imi_exec, + [INSTR_TABLE] = instr_table_exec, [INSTR_EXTERN_OBJ] = instr_extern_obj_exec, [INSTR_EXTERN_FUNC] = instr_extern_func_exec, @@ -8048,6 +8992,182 @@ regarray_free(struct rte_swx_pipeline *p) } } +/* + * Meter array. + */ +static struct meter_profile * +meter_profile_find(struct rte_swx_pipeline *p, const char *name) +{ + struct meter_profile *elem; + + TAILQ_FOREACH(elem, &p->meter_profiles, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct metarray * +metarray_find(struct rte_swx_pipeline *p, const char *name) +{ + struct metarray *elem; + + TAILQ_FOREACH(elem, &p->metarrays, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct metarray * +metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id) +{ + struct metarray *elem = NULL; + + TAILQ_FOREACH(elem, &p->metarrays, node) + if (elem->id == id) + return elem; + + return NULL; +} + +int +rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size) +{ + struct metarray *m; + + CHECK(p, EINVAL); + + CHECK_NAME(name, EINVAL); + CHECK(!metarray_find(p, name), EEXIST); + + CHECK(size, EINVAL); + size = rte_align32pow2(size); + + /* Memory allocation. */ + m = calloc(1, sizeof(struct metarray)); + CHECK(m, ENOMEM); + + /* Node initialization. */ + strcpy(m->name, name); + m->size = size; + m->id = p->n_metarrays; + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->metarrays, m, node); + p->n_metarrays++; + + return 0; +} + +struct meter_profile meter_profile_default = { + .node = {0}, + .name = "", + .params = {0}, + + .profile = { + .cbs = 10000, + .pbs = 10000, + .cir_period = 1, + .cir_bytes_per_period = 1, + .pir_period = 1, + .pir_bytes_per_period = 1, + }, + + .n_users = 0, +}; + +static void +meter_init(struct meter *m) +{ + memset(m, 0, sizeof(struct meter)); + rte_meter_trtcm_config(&m->m, &meter_profile_default.profile); + m->profile = &meter_profile_default; + m->color_mask = RTE_COLOR_GREEN; + + meter_profile_default.n_users++; +} + +static int +metarray_build(struct rte_swx_pipeline *p) +{ + struct metarray *m; + + if (!p->n_metarrays) + return 0; + + p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime)); + CHECK(p->metarray_runtime, ENOMEM); + + TAILQ_FOREACH(m, &p->metarrays, node) { + struct metarray_runtime *r = &p->metarray_runtime[m->id]; + uint32_t i; + + r->metarray = env_malloc(m->size * sizeof(struct meter), + RTE_CACHE_LINE_SIZE, + p->numa_node); + CHECK(r->metarray, ENOMEM); + + for (i = 0; i < m->size; i++) + meter_init(&r->metarray[i]); + + r->size_mask = m->size - 1; + } + + return 0; +} + +static void +metarray_build_free(struct rte_swx_pipeline *p) +{ + uint32_t i; + + if (!p->metarray_runtime) + return; + + for (i = 0; i < p->n_metarrays; i++) { + struct metarray *m = metarray_find_by_id(p, i); + struct metarray_runtime *r = &p->metarray_runtime[i]; + + env_free(r->metarray, m->size * sizeof(struct meter)); + } + + free(p->metarray_runtime); + p->metarray_runtime = NULL; +} + +static void +metarray_free(struct rte_swx_pipeline *p) +{ + metarray_build_free(p); + + /* Meter arrays. */ + for ( ; ; ) { + struct metarray *elem; + + elem = TAILQ_FIRST(&p->metarrays); + if (!elem) + break; + + TAILQ_REMOVE(&p->metarrays, elem, node); + free(elem); + } + + /* Meter profiles. */ + for ( ; ; ) { + struct meter_profile *elem; + + elem = TAILQ_FIRST(&p->meter_profiles); + if (!elem) + break; + + TAILQ_REMOVE(&p->meter_profiles, elem, node); + free(elem); + } +} + /* * Pipeline. */ @@ -8077,6 +9197,8 @@ rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node) TAILQ_INIT(&pipeline->table_types); TAILQ_INIT(&pipeline->tables); TAILQ_INIT(&pipeline->regarrays); + TAILQ_INIT(&pipeline->meter_profiles); + TAILQ_INIT(&pipeline->metarrays); pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */ pipeline->numa_node = numa_node; @@ -8093,6 +9215,7 @@ rte_swx_pipeline_free(struct rte_swx_pipeline *p) free(p->instructions); + metarray_free(p); regarray_free(p); table_state_free(p); table_free(p); @@ -8182,10 +9305,15 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p) if (status) goto error; + status = metarray_build(p); + if (status) + goto error; + p->build_done = 1; return 0; error: + metarray_build_free(p); regarray_build_free(p); table_state_build_free(p); table_build_free(p); @@ -8248,6 +9376,7 @@ rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p, pipeline->n_actions = n_actions; pipeline->n_tables = n_tables; pipeline->n_regarrays = p->n_regarrays; + pipeline->n_metarrays = p->n_metarrays; return 0; } @@ -8516,3 +9645,166 @@ rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, r->regarray[regarray_index] = value; return 0; } + +int +rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p, + uint32_t metarray_id, + struct rte_swx_ctl_metarray_info *metarray) +{ + struct metarray *m; + + if (!p || !metarray) + return -EINVAL; + + m = metarray_find_by_id(p, metarray_id); + if (!m) + return -EINVAL; + + strcpy(metarray->name, m->name); + metarray->size = m->size; + return 0; +} + +int +rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p, + const char *name, + struct rte_meter_trtcm_params *params) +{ + struct meter_profile *mp; + int status; + + CHECK(p, EINVAL); + CHECK_NAME(name, EINVAL); + CHECK(params, EINVAL); + CHECK(!meter_profile_find(p, name), EEXIST); + + /* Node allocation. */ + mp = calloc(1, sizeof(struct meter_profile)); + CHECK(mp, ENOMEM); + + /* Node initialization. */ + strcpy(mp->name, name); + memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params)); + status = rte_meter_trtcm_profile_config(&mp->profile, params); + if (status) { + free(mp); + CHECK(0, EINVAL); + } + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node); + + return 0; +} + +int +rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p, + const char *name) +{ + struct meter_profile *mp; + + CHECK(p, EINVAL); + CHECK_NAME(name, EINVAL); + + mp = meter_profile_find(p, name); + CHECK(mp, EINVAL); + CHECK(!mp->n_users, EBUSY); + + /* Remove node from tailq. */ + TAILQ_REMOVE(&p->meter_profiles, mp, node); + free(mp); + + return 0; +} + +int +rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index) +{ + struct meter_profile *mp_old; + struct metarray *metarray; + struct metarray_runtime *metarray_runtime; + struct meter *m; + + CHECK(p, EINVAL); + CHECK_NAME(metarray_name, EINVAL); + + metarray = metarray_find(p, metarray_name); + CHECK(metarray, EINVAL); + CHECK(metarray_index < metarray->size, EINVAL); + + metarray_runtime = &p->metarray_runtime[metarray->id]; + m = &metarray_runtime->metarray[metarray_index]; + mp_old = m->profile; + + meter_init(m); + + mp_old->n_users--; + + return 0; +} + +int +rte_swx_ctl_meter_set(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + const char *profile_name) +{ + struct meter_profile *mp, *mp_old; + struct metarray *metarray; + struct metarray_runtime *metarray_runtime; + struct meter *m; + + CHECK(p, EINVAL); + CHECK_NAME(metarray_name, EINVAL); + + metarray = metarray_find(p, metarray_name); + CHECK(metarray, EINVAL); + CHECK(metarray_index < metarray->size, EINVAL); + + mp = meter_profile_find(p, profile_name); + CHECK(mp, EINVAL); + + metarray_runtime = &p->metarray_runtime[metarray->id]; + m = &metarray_runtime->metarray[metarray_index]; + mp_old = m->profile; + + memset(m, 0, sizeof(struct meter)); + rte_meter_trtcm_config(&m->m, &mp->profile); + m->profile = mp; + m->color_mask = RTE_COLORS; + + mp->n_users++; + mp_old->n_users--; + + return 0; +} + +int +rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p, + const char *metarray_name, + uint32_t metarray_index, + struct rte_swx_ctl_meter_stats *stats) +{ + struct metarray *metarray; + struct metarray_runtime *metarray_runtime; + struct meter *m; + + CHECK(p, EINVAL); + CHECK_NAME(metarray_name, EINVAL); + + metarray = metarray_find(p, metarray_name); + CHECK(metarray, EINVAL); + CHECK(metarray_index < metarray->size, EINVAL); + + CHECK(stats, EINVAL); + + metarray_runtime = &p->metarray_runtime[metarray->id]; + m = &metarray_runtime->metarray[metarray_index]; + + memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts)); + memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes)); + + return 0; +} diff --git a/lib/librte_pipeline/rte_swx_pipeline.h b/lib/librte_pipeline/rte_swx_pipeline.h index 1c9b2eb12..156b1fd67 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.h +++ b/lib/librte_pipeline/rte_swx_pipeline.h @@ -640,6 +640,28 @@ rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, uint32_t size, uint64_t init_val); +/** + * Pipeline meter array configure + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Meter array name. + * @param[in] size + * Number of meters in the array. Each meter in the array implements the Two + * Rate Three Color Marker (trTCM) algorithm, as specified by RFC 2698. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Meter array with this name already exists. + */ +__rte_experimental +int +rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size); + /** * Pipeline instructions configure * diff --git a/lib/librte_pipeline/rte_swx_pipeline_spec.c b/lib/librte_pipeline/rte_swx_pipeline_spec.c index 0d0bca2ed..2e867d7bf 100644 --- a/lib/librte_pipeline/rte_swx_pipeline_spec.c +++ b/lib/librte_pipeline/rte_swx_pipeline_spec.c @@ -1015,6 +1015,68 @@ regarray_statement_parse(struct regarray_spec *s, return 0; } +/* + * metarray. + * + * metarray NAME size SIZE + */ +struct metarray_spec { + char *name; + uint32_t size; +}; + +static void +metarray_spec_free(struct metarray_spec *s) +{ + if (!s) + return; + + free(s->name); + s->name = NULL; +} + +static int +metarray_statement_parse(struct metarray_spec *s, + char **tokens, + uint32_t n_tokens, + uint32_t n_lines, + uint32_t *err_line, + const char **err_msg) +{ + char *p; + + /* Check format. */ + if ((n_tokens != 4) || strcmp(tokens[2], "size")) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid metarray statement."; + return -EINVAL; + } + + /* spec. */ + s->name = strdup(tokens[1]); + if (!s->name) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Memory allocation failed."; + return -ENOMEM; + } + + p = tokens[3]; + s->size = strtoul(p, &p, 0); + if (p[0] || !s->size) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid size argument."; + return -EINVAL; + } + + return 0; +} + /* * apply. * @@ -1142,6 +1204,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, struct action_spec action_spec = {0}; struct table_spec table_spec = {0}; struct regarray_spec regarray_spec = {0}; + struct metarray_spec metarray_spec = {0}; struct apply_spec apply_spec = {0}; uint32_t n_lines; uint32_t block_mask = 0; @@ -1509,6 +1572,33 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, continue; } + /* metarray. */ + if (!strcmp(tokens[0], "metarray")) { + status = metarray_statement_parse(&metarray_spec, + tokens, + n_tokens, + n_lines, + err_line, + err_msg); + if (status) + goto error; + + status = rte_swx_pipeline_metarray_config(p, + metarray_spec.name, + metarray_spec.size); + if (status) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Meter array configuration error."; + goto error; + } + + metarray_spec_free(&metarray_spec); + + continue; + } + /* apply. */ if (!strcmp(tokens[0], "apply")) { status = apply_statement_parse(&block_mask, @@ -1562,6 +1652,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, action_spec_free(&action_spec); table_spec_free(&table_spec); regarray_spec_free(®array_spec); + metarray_spec_free(&metarray_spec); apply_spec_free(&apply_spec); return status; } diff --git a/lib/librte_pipeline/version.map b/lib/librte_pipeline/version.map index 26a9edf47..2049d6d69 100644 --- a/lib/librte_pipeline/version.map +++ b/lib/librte_pipeline/version.map @@ -104,8 +104,15 @@ EXPERIMENTAL { rte_swx_pipeline_table_type_register; #added in 21.05 + rte_swx_ctl_metarray_info_get; + rte_swx_ctl_meter_profile_add; + rte_swx_ctl_meter_profile_delete; + rte_swx_ctl_meter_reset; + rte_swx_ctl_meter_set; + rte_swx_ctl_meter_stats_read; rte_swx_ctl_pipeline_regarray_read; rte_swx_ctl_pipeline_regarray_write; rte_swx_ctl_regarray_info_get; + rte_swx_pipeline_metarray_config; rte_swx_pipeline_regarray_config; }; -- 2.17.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [dpdk-dev] [PATCH v5 2/2] pipeline: add meter support to the SWX pipeline 2021-03-16 11:40 ` [dpdk-dev] [PATCH v5 2/2] pipeline: add meter support to the " Cristian Dumitrescu @ 2021-03-24 18:19 ` Thomas Monjalon 0 siblings, 0 replies; 8+ messages in thread From: Thomas Monjalon @ 2021-03-24 18:19 UTC (permalink / raw) To: Cristian Dumitrescu; +Cc: dev 16/03/2021 12:40, Cristian Dumitrescu: > Meter arrays are stateful objects that are updated by the data plane > and configured & monitored by the control plane. The meters implement > the RFC 2698 Two Rate Three Color Marker (trTCM) algorithm. > > Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com> Series applied, thanks ^ permalink raw reply [flat|nested] 8+ messages in thread
end of thread, other threads:[~2021-03-24 18:19 UTC | newest] Thread overview: 8+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2021-02-26 20:26 [dpdk-dev] [PATCH v2] pipeline: add register array support to SWX pipeline Cristian Dumitrescu 2021-03-15 22:22 ` [dpdk-dev] [PATCH v3 1/2] " Cristian Dumitrescu 2021-03-15 22:22 ` [dpdk-dev] [PATCH v3 2/2] pipeline: add meter support to the " Cristian Dumitrescu 2021-03-15 22:48 ` [dpdk-dev] [PATCH v4 1/2] pipeline: add register array support to " Cristian Dumitrescu 2021-03-15 22:48 ` [dpdk-dev] [PATCH v4 2/2] pipeline: add meter support to the " Cristian Dumitrescu 2021-03-16 11:40 ` [dpdk-dev] [PATCH v5 1/2] pipeline: add register array support to " Cristian Dumitrescu 2021-03-16 11:40 ` [dpdk-dev] [PATCH v5 2/2] pipeline: add meter support to the " Cristian Dumitrescu 2021-03-24 18:19 ` Thomas Monjalon
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).