From: Yuanhan Liu <yuanhan.liu@linux.intel.com>
To: dev@dpdk.org
Cc: Maxime Coquelin <maxime.coquelin@redhat.com>,
Harris James R <james.r.harris@intel.com>,
Liu Changpeng <changpeng.liu@intel.com>,
Yuanhan Liu <yuanhan.liu@linux.intel.com>
Subject: [dpdk-dev] [PATCH v3 17/22] vhost: export APIs for live migration support
Date: Tue, 28 Mar 2017 20:45:37 +0800 [thread overview]
Message-ID: <1490705142-893-18-git-send-email-yuanhan.liu@linux.intel.com> (raw)
In-Reply-To: <1490705142-893-1-git-send-email-yuanhan.liu@linux.intel.com>
Export few APIs for the vhost-user driver to log the guest memory writes,
which is a must for live migration support.
This patch basically moves vhost_log_write() and vhost_log_used_vring()
into vhost.h and then add an wrapper (the public API) to them.
Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
---
lib/librte_vhost/rte_vhost_version.map | 2 ++
lib/librte_vhost/rte_virtio_net.h | 43 ++++++++++++++++++++++++++++++++++
lib/librte_vhost/vhost.c | 31 ++++++++++++++++++++++++
lib/librte_vhost/vhost.h | 38 ++++++++++++++++++++++++++++++
lib/librte_vhost/virtio_net.c | 36 ----------------------------
5 files changed, 114 insertions(+), 36 deletions(-)
diff --git a/lib/librte_vhost/rte_vhost_version.map b/lib/librte_vhost/rte_vhost_version.map
index 8df14dc..f4b74da 100644
--- a/lib/librte_vhost/rte_vhost_version.map
+++ b/lib/librte_vhost/rte_vhost_version.map
@@ -41,5 +41,7 @@ DPDK_17.05 {
rte_vhost_get_vhost_vring;
rte_vhost_get_vring_num;
rte_vhost_gpa_to_vva;
+ rte_vhost_log_used_vring;
+ rte_vhost_log_write;
} DPDK_16.07;
diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h
index 4256927..11b204d 100644
--- a/lib/librte_vhost/rte_virtio_net.h
+++ b/lib/librte_vhost/rte_virtio_net.h
@@ -132,6 +132,49 @@ static inline uint64_t __attribute__((always_inline))
return 0;
}
+#define RTE_VHOST_NEED_LOG(features) ((features) & (1ULL << VHOST_F_LOG_ALL))
+
+/**
+ * Log the memory write start with given address.
+ *
+ * This function only need be invoked when the live migration starts.
+ * Therefore, we won't need call it at all in the most of time. For
+ * making the performance impact be minimum, it's suggested to do a
+ * check before calling it:
+ *
+ * if (unlikely(RTE_VHOST_NEED_LOG(features)))
+ * rte_vhost_log_write(vid, addr, len);
+ *
+ * @param vid
+ * vhost device ID
+ * @param addr
+ * the starting address for write
+ * @param len
+ * the length to write
+ */
+void rte_vhost_log_write(int vid, uint64_t addr, uint64_t len);
+
+/**
+ * Log the used ring update start at given offset.
+ *
+ * Same as rte_vhost_log_write, it's suggested to do a check before
+ * calling it:
+ *
+ * if (unlikely(RTE_VHOST_NEED_LOG(features)))
+ * rte_vhost_log_used_vring(vid, vring_idx, offset, len);
+ *
+ * @param vid
+ * vhost device ID
+ * @param vring_idx
+ * the vring index
+ * @param offset
+ * the offset inside the used ring
+ * @param len
+ * the length to write
+ */
+void rte_vhost_log_used_vring(int vid, uint16_t vring_idx,
+ uint64_t offset, uint64_t len);
+
int rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable);
/**
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index 74ae3b2..8be5b6a 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -443,3 +443,34 @@ struct virtio_net *
dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
return 0;
}
+
+void
+rte_vhost_log_write(int vid, uint64_t addr, uint64_t len)
+{
+ struct virtio_net *dev = get_device(vid);
+
+ if (dev == NULL)
+ return;
+
+ vhost_log_write(dev, addr, len);
+}
+
+void
+rte_vhost_log_used_vring(int vid, uint16_t vring_idx,
+ uint64_t offset, uint64_t len)
+{
+ struct virtio_net *dev;
+ struct vhost_virtqueue *vq;
+
+ dev = get_device(vid);
+ if (dev == NULL)
+ return;
+
+ if (vring_idx >= VHOST_MAX_VRING)
+ return;
+ vq = dev->virtqueue[vring_idx];
+ if (!vq)
+ return;
+
+ vhost_log_used_vring(dev, vq, offset, len);
+}
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 225ff2e..a199ee6 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -198,6 +198,44 @@ struct virtio_net {
struct guest_page *guest_pages;
} __rte_cache_aligned;
+
+#define VHOST_LOG_PAGE 4096
+
+static inline void __attribute__((always_inline))
+vhost_log_page(uint8_t *log_base, uint64_t page)
+{
+ log_base[page / 8] |= 1 << (page % 8);
+}
+
+static inline void __attribute__((always_inline))
+vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
+{
+ uint64_t page;
+
+ if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) ||
+ !dev->log_base || !len))
+ return;
+
+ if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
+ return;
+
+ /* To make sure guest memory updates are committed before logging */
+ rte_smp_wmb();
+
+ page = addr / VHOST_LOG_PAGE;
+ while (page * VHOST_LOG_PAGE < addr + len) {
+ vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
+ page += 1;
+ }
+}
+
+static inline void __attribute__((always_inline))
+vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq,
+ uint64_t offset, uint64_t len)
+{
+ vhost_log_write(dev, vq->log_guest_addr + offset, len);
+}
+
/* Macros for printing using RTE_LOG */
#define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1
#define RTE_LOGTYPE_VHOST_DATA RTE_LOGTYPE_USER1
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 8ed2b93..7ae7904 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -48,42 +48,6 @@
#include "vhost.h"
#define MAX_PKT_BURST 32
-#define VHOST_LOG_PAGE 4096
-
-static inline void __attribute__((always_inline))
-vhost_log_page(uint8_t *log_base, uint64_t page)
-{
- log_base[page / 8] |= 1 << (page % 8);
-}
-
-static inline void __attribute__((always_inline))
-vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
-{
- uint64_t page;
-
- if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) ||
- !dev->log_base || !len))
- return;
-
- if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
- return;
-
- /* To make sure guest memory updates are committed before logging */
- rte_smp_wmb();
-
- page = addr / VHOST_LOG_PAGE;
- while (page * VHOST_LOG_PAGE < addr + len) {
- vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
- page += 1;
- }
-}
-
-static inline void __attribute__((always_inline))
-vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq,
- uint64_t offset, uint64_t len)
-{
- vhost_log_write(dev, vq->log_guest_addr + offset, len);
-}
static bool
is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring)
--
1.9.0
next prev parent reply other threads:[~2017-03-28 12:48 UTC|newest]
Thread overview: 135+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-03-03 9:51 [dpdk-dev] [PATCH 00/17] vhost: generic vhost API Yuanhan Liu
2017-03-03 9:51 ` [dpdk-dev] [PATCH 01/17] vhost: introduce driver features related APIs Yuanhan Liu
2017-03-14 9:46 ` Maxime Coquelin
2017-03-14 9:53 ` Maxime Coquelin
2017-03-16 7:08 ` Yuanhan Liu
2017-03-16 9:18 ` Maxime Coquelin
2017-03-17 5:50 ` Yuanhan Liu
2017-03-03 9:51 ` [dpdk-dev] [PATCH 02/17] net/vhost: remove feature " Yuanhan Liu
2017-03-14 10:15 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 03/17] vhost: use new APIs to handle features Yuanhan Liu
2017-03-14 10:43 ` Maxime Coquelin
2017-03-16 7:43 ` Yuanhan Liu
2017-03-16 9:39 ` Maxime Coquelin
2017-03-17 5:48 ` Yuanhan Liu
2017-03-03 9:51 ` [dpdk-dev] [PATCH 04/17] vhost: make notify ops per vhost driver Yuanhan Liu
2017-03-14 10:55 ` Maxime Coquelin
2017-03-16 7:50 ` Yuanhan Liu
2017-03-03 9:51 ` [dpdk-dev] [PATCH 05/17] vhost: export guest memory regions Yuanhan Liu
2017-03-14 11:00 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 06/17] vhost: introduce API to fetch negotiated features Yuanhan Liu
2017-03-14 11:02 ` Maxime Coquelin
2017-03-16 7:35 ` Yuanhan Liu
2017-03-16 9:22 ` Maxime Coquelin
2017-03-17 5:49 ` Yuanhan Liu
2017-03-03 9:51 ` [dpdk-dev] [PATCH 07/17] vhost: export vhost vring info Yuanhan Liu
2017-03-14 12:11 ` Maxime Coquelin
2017-03-16 7:24 ` Yuanhan Liu
2017-03-16 9:20 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 08/17] vhost: export API to translate gpa to vva Yuanhan Liu
2017-03-14 12:24 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 09/17] vhost: turn queue pair to vring Yuanhan Liu
2017-03-14 12:31 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 10/17] vhost: export the number of vrings Yuanhan Liu
2017-03-14 12:33 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 11/17] vhost: move the device ready check at proper place Yuanhan Liu
2017-03-14 12:37 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 12/17] vhost: drop the Rx and Tx queue macro Yuanhan Liu
2017-03-14 12:42 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 13/17] vhost: do not include net specific headers Yuanhan Liu
2017-03-14 12:46 ` Maxime Coquelin
2017-03-20 7:32 ` Liu, Changpeng
2017-03-22 6:21 ` Yuanhan Liu
2017-03-03 9:51 ` [dpdk-dev] [PATCH 14/17] vhost: rename device ops struct Yuanhan Liu
2017-03-14 12:48 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 15/17] vhost: rename virtio-net to vhost Yuanhan Liu
2017-03-14 12:50 ` Maxime Coquelin
2017-03-03 9:51 ` [dpdk-dev] [PATCH 16/17] vhost: rename header file Yuanhan Liu
2017-03-14 12:59 ` Maxime Coquelin
2017-03-20 5:35 ` Yuanhan Liu
2017-03-03 9:51 ` [dpdk-dev] [PATCH 17/17] examples/vhost: demonstrate the new generic vhost APIs Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 00/22] vhost: generic vhost API Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 01/22] vhost: introduce driver features related APIs Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 02/22] net/vhost: remove feature " Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 03/22] vhost: use new APIs to handle features Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 04/22] vhost: make notify ops per vhost driver Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 05/22] vhost: export guest memory regions Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 06/22] vhost: introduce API to fetch negotiated features Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 07/22] vhost: export vhost vring info Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 08/22] vhost: export API to translate gpa to vva Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 09/22] vhost: turn queue pair to vring Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 10/22] vhost: export the number of vrings Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 11/22] vhost: move the device ready check at proper place Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 12/22] vhost: drop the Rx and Tx queue macro Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 13/22] vhost: do not include net specific headers Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 14/22] vhost: rename device ops struct Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 15/22] vhost: rename virtio-net to vhost Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 16/22] vhost: add features changed callback Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 17/22] vhost: export APIs for live migration support Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 18/22] vhost: introduce API to start a specific driver Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 19/22] vhost: rename header file Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 20/22] vhost: workaround the build dependency on mbuf header Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 21/22] vhost: do not destroy device on repeat mem table message Yuanhan Liu
2017-03-23 7:10 ` [dpdk-dev] [PATCH v2 22/22] examples/vhost: demonstrate the new generic vhost APIs Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 00/22] vhost: generic vhost API Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 01/22] vhost: introduce driver features related APIs Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 02/22] net/vhost: remove feature " Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 03/22] vhost: use new APIs to handle features Yuanhan Liu
2017-03-29 14:57 ` Maxime Coquelin
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 04/22] vhost: make notify ops per vhost driver Yuanhan Liu
2017-03-29 15:03 ` Maxime Coquelin
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 05/22] vhost: export guest memory regions Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 06/22] vhost: introduce API to fetch negotiated features Yuanhan Liu
2017-03-31 7:45 ` Maxime Coquelin
2017-03-31 8:51 ` Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 07/22] vhost: export vhost vring info Yuanhan Liu
2017-03-31 7:48 ` Maxime Coquelin
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 08/22] vhost: export API to translate gpa to vva Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 09/22] vhost: turn queue pair to vring Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 10/22] vhost: export the number of vrings Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 11/22] vhost: move the device ready check at proper place Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 12/22] vhost: drop the Rx and Tx queue macro Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 13/22] vhost: do not include net specific headers Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 14/22] vhost: rename device ops struct Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 15/22] vhost: rename virtio-net to vhost Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 16/22] vhost: add features changed callback Yuanhan Liu
2017-03-31 7:50 ` Maxime Coquelin
2017-03-28 12:45 ` Yuanhan Liu [this message]
2017-03-31 8:05 ` [dpdk-dev] [PATCH v3 17/22] vhost: export APIs for live migration support Maxime Coquelin
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 18/22] vhost: introduce API to start a specific driver Yuanhan Liu
2017-03-31 9:11 ` Maxime Coquelin
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 19/22] vhost: rename header file Yuanhan Liu
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 20/22] vhost: workaround the build dependency on mbuf header Yuanhan Liu
2017-03-31 9:13 ` Maxime Coquelin
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 21/22] vhost: do not destroy device on repeat mem table message Yuanhan Liu
2017-03-31 9:26 ` Maxime Coquelin
2017-03-28 12:45 ` [dpdk-dev] [PATCH v3 22/22] examples/vhost: demonstrate the new generic vhost APIs Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 00/22] vhost: generic vhost API Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 01/22] vhost: introduce driver features related APIs Yuanhan Liu
2017-04-05 0:01 ` Thomas Monjalon
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 02/22] net/vhost: remove feature " Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 03/22] vhost: use new APIs to handle features Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 04/22] vhost: make notify ops per vhost driver Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 05/22] vhost: export guest memory regions Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 06/22] vhost: introduce API to fetch negotiated features Yuanhan Liu
2017-04-01 8:28 ` Maxime Coquelin
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 07/22] vhost: export vhost vring info Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 08/22] vhost: export API to translate gpa to vva Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 09/22] vhost: turn queue pair to vring Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 10/22] vhost: export the number of vrings Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 11/22] vhost: move the device ready check at proper place Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 12/22] vhost: drop the Rx and Tx queue macro Yuanhan Liu
2017-04-05 0:17 ` Thomas Monjalon
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 13/22] vhost: do not include net specific headers Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 14/22] vhost: rename device ops struct Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 15/22] vhost: rename virtio-net to vhost Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 16/22] vhost: add features changed callback Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 17/22] vhost: export APIs for live migration support Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 18/22] vhost: introduce API to start a specific driver Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 19/22] vhost: rename header file Yuanhan Liu
2017-04-05 0:26 ` Thomas Monjalon
2017-04-05 5:16 ` Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 20/22] vhost: workaround the build dependency on mbuf header Yuanhan Liu
2017-04-01 7:22 ` [dpdk-dev] [PATCH v4 21/22] vhost: do not destroy device on repeat mem table message Yuanhan Liu
2017-04-01 7:23 ` [dpdk-dev] [PATCH v4 22/22] examples/vhost: demonstrate the new generic vhost APIs Yuanhan Liu
2017-04-01 8:44 ` [dpdk-dev] [PATCH v4 00/22] vhost: generic vhost API Yuanhan Liu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1490705142-893-18-git-send-email-yuanhan.liu@linux.intel.com \
--to=yuanhan.liu@linux.intel.com \
--cc=changpeng.liu@intel.com \
--cc=dev@dpdk.org \
--cc=james.r.harris@intel.com \
--cc=maxime.coquelin@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).