Skip to content
This repository
Browse code

jsonstats: Adds "JSONStats" component

JSONStats acts as a proxy for OpenFlow requests. It accepts JSON-formatted
messages and sends the equivalent request to connected datapaths, then relays
the replies back in JSON.

Signed-off-by: Joe Stringer <joestringernz@gmail.com>
  • Loading branch information...
commit 20cb3ff3b47d65d7aa31b3567c4408065a3635a5 1 parent cc7cdc8
Joe Stringer authored May 29, 2012
2  nox/configure.ac.in
@@ -68,7 +68,7 @@ ACI_PACKAGE([netapps],[misc network apps],
68 68
                	routing user_event_log tests topology discovery 
69 69
                	bindings_storage switchstats flow_fetcher data
70 70
                	switch_management networkstate hoststate tablog 
71  
-               	route lavi rfproxy
  71
+               	route lavi rfproxy jsonstats
72 72
                 #add netapps component here
73 73
 		],
74 74
                [TURN_ON_NETAPPS])
23  nox/src/nox/netapps/jsonstats/Makefile.am
... ...
@@ -0,0 +1,23 @@
  1
+include ../../../Make.vars
  2
+
  3
+CONFIGURE_DEPENCIES = $(srcdir)/Makefile.am
  4
+
  5
+EXTRA_DIST =\
  6
+	meta.json
  7
+
  8
+pkglib_LTLIBRARIES = jsonstats.la
  9
+
  10
+jsonstats_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/nox \
  11
+                                    -I$(top_srcdir)/src/nox/netapps/jsonstats \
  12
+                                    -I$(top_srcdir)/src/nox/coreapps/messenger \
  13
+                                    -I$(top_srcdir)/src/nox/netapps \
  14
+                                    -I$(top_srcdir)/../include
  15
+
  16
+jsonstats_la_SOURCES = stats_stream.cc jsonstats.cc
  17
+jsonstats_la_LDFLAGS = -module -export-dynamic
  18
+
  19
+NOX_RUNTIMEFILES = meta.json
  20
+
  21
+all-local: nox-all-local
  22
+clean-local: nox-clean-local
  23
+install-exec-hook: nox-install-local
400  nox/src/nox/netapps/jsonstats/jsonstats.cc
... ...
@@ -0,0 +1,400 @@
  1
+/*
  2
+ *  Copyright 2012 Fundação CPqD
  3
+ *
  4
+ *   Licensed under the Apache License, Version 2.0 (the "License");
  5
+ *   you may not use this file except in compliance with the License.
  6
+ *   You may obtain a copy of the License at
  7
+ *
  8
+ *      http://www.apache.org/licenses/LICENSE-2.0
  9
+ *
  10
+ *   Unless required by applicable law or agreed to in writing, software
  11
+ *   distributed under the License is distributed on an "AS IS" BASIS,
  12
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13
+ *   See the License for the specific language governing permissions and
  14
+ *   limitations under the License.
  15
+ */
  16
+#include <boost/bind.hpp>
  17
+#include <map>
  18
+#include "assert.hh"
  19
+#include "component.hh"
  20
+#include "vlog.hh"
  21
+#include "datapath-join.hh"
  22
+#include "datapath-leave.hh"
  23
+#include "port-stats-in.hh"
  24
+#include "json-util.hh"
  25
+#include "jsonmessenger.hh"
  26
+#include "timeval.hh"
  27
+#include "netinet++/datapathid.hh"
  28
+#include "stats_stream.hh"
  29
+#include <sys/types.h>
  30
+#include <stdlib.h>
  31
+#include <unistd.h>
  32
+#include <fcntl.h>
  33
+#include <iostream>
  34
+#include <sstream>
  35
+
  36
+namespace {
  37
+
  38
+using namespace vigil;
  39
+using namespace vigil::container;
  40
+using std::map;
  41
+using std::pair;
  42
+using std::vector;
  43
+using std::list;
  44
+using std::string;
  45
+
  46
+Vlog_module lg("jsonstats");
  47
+
  48
+const uint32_t DEFAULT_RESPONSE_TIMEOUT = 500; /* milliseconds */
  49
+
  50
+/*
  51
+ * This NOX app recieves requests for statistics using jsonmessenger.
  52
+ * Statistics are retrieved, collated and sent back to interested clients in an
  53
+ * openflow-esque JSON format.
  54
+ */
  55
+class JSONStats: public Component {
  56
+  int response_timeout_;
  57
+  hash_map<datapathid,vector<Port> > datapaths_;
  58
+  Stats_stream *streams_[SPRT_MAX];
  59
+public:
  60
+  JSONStats(const Context *c, const json_object*) : Component(c) {
  61
+    int i;
  62
+    for (i = 0; i < SPRT_MAX; ++i)
  63
+      streams_[i] = NULL;
  64
+
  65
+    response_timeout_ = DEFAULT_RESPONSE_TIMEOUT;
  66
+  }
  67
+
  68
+  void configure(const Configuration*);
  69
+  void install();
  70
+
  71
+  void request_features();
  72
+  void request_port_stats();
  73
+  void finalise_stream(int request);
  74
+  void add_client(stats_pkt_request_type request, Msg_stream *sock);
  75
+  void copy_port_information(struct vector<Port> &ports,
  76
+                             const Datapath_join_event& dj);
  77
+
  78
+  Disposition handle_datapath_join(const Event& e);
  79
+  Disposition handle_datapath_leave(const Event& e);
  80
+  Disposition handle_ports_stats(const Event& e);
  81
+  Disposition handle_json_event(const Event& e);
  82
+};
  83
+
  84
+void create_port_stats_request(ofp_stats_request *osr, size_t size) {
  85
+  osr->header.type = OFPT_STATS_REQUEST;
  86
+  osr->header.version = OFP_VERSION;
  87
+  osr->header.length = htons(size);
  88
+  osr->header.xid = htonl(0);
  89
+  osr->type = htons(OFPST_PORT);
  90
+  osr->flags = htons(0);
  91
+
  92
+  struct ofp_port_stats_request *psr = (ofp_port_stats_request *)(osr->body);
  93
+  psr->port_no = htons(OFPP_NONE);
  94
+}
  95
+
  96
+/**
  97
+ * TODO: Extract configuration items from config object
  98
+ *
  99
+ * "What configuration items?", you say?
  100
+ * - jsonmessenger listen port
  101
+ * - response timeout value
  102
+ */
  103
+void JSONStats::configure(const Configuration* config) {
  104
+}
  105
+
  106
+void JSONStats::install() {
  107
+  /* Register with NOX for events we're interested in */
  108
+  register_handler<JSONMsg_event> (boost::bind(
  109
+      &JSONStats::handle_json_event, this, _1));
  110
+  register_handler<Datapath_join_event> (boost::bind(
  111
+      &JSONStats::handle_datapath_join, this, _1));
  112
+  register_handler<Datapath_leave_event> (boost::bind(
  113
+      &JSONStats::handle_datapath_leave, this, _1));
  114
+  register_handler<Port_stats_in_event> (boost::bind(
  115
+      &JSONStats::handle_ports_stats, this, _1));
  116
+}
  117
+
  118
+/**
  119
+ * Note: We don't check to see if each datapath will support the stats request.
  120
+ */
  121
+void JSONStats::request_port_stats() {
  122
+  size_t msize = sizeof(ofp_stats_request) + sizeof(ofp_port_stats_request);
  123
+  boost::shared_array<uint8_t> raw_sr(new uint8_t[msize]);
  124
+  ofp_stats_request *osr = (ofp_stats_request *)raw_sr.get();
  125
+  create_port_stats_request(osr, msize);
  126
+
  127
+  /* Send OpenFlow Stats Request to all registered datapaths. */
  128
+  hash_map<datapathid,vector<Port> >::iterator iter;
  129
+  for (iter = datapaths_.begin(); iter != datapaths_.end(); ++iter) {
  130
+    VLOG_DBG(lg, "Sent ofp_stats-request to 0x%lx", iter->first.as_host());
  131
+    send_openflow_command(iter->first, &osr->header, false);
  132
+  }
  133
+}
  134
+
  135
+void JSONStats::copy_port_information(vector<Port> &ports,
  136
+                                      const Datapath_join_event& dj) {
  137
+  ports.reserve(dj.ports.size());
  138
+  vector<Port>::const_iterator iter = dj.ports.begin();
  139
+  for (; iter != dj.ports.end(); ++iter) {
  140
+    ports.push_back(*iter);
  141
+  }
  142
+}
  143
+
  144
+Disposition JSONStats::handle_datapath_join(const Event& e) {
  145
+  const Datapath_join_event& dj = assert_cast<const Datapath_join_event&> (e);
  146
+
  147
+  hash_map<datapathid,vector<Port> >::iterator iter =
  148
+      datapaths_.find(dj.datapath_id);
  149
+  if (iter != datapaths_.end()) {
  150
+    VLOG_INFO(lg, "Duplicate datapath join ignored: 0x%lx",
  151
+              dj.datapath_id.as_host());
  152
+    return CONTINUE;
  153
+  }
  154
+
  155
+  /* Add the datapath and its set of Port features to the hash_map. */
  156
+  vector<Port> ports;
  157
+  copy_port_information(ports, dj);
  158
+  datapaths_.insert( pair<datapathid,vector<Port> >(dj.datapath_id, ports));
  159
+
  160
+  VLOG_DBG(lg, "DP joined: 0x%lx", dj.datapath_id.as_host());
  161
+
  162
+  return CONTINUE;
  163
+}
  164
+
  165
+Disposition JSONStats::handle_datapath_leave(const Event& e) {
  166
+  const Datapath_leave_event& dl = assert_cast<const Datapath_leave_event&> (e);
  167
+
  168
+  datapaths_.erase(dl.datapath_id);
  169
+  VLOG_DBG(lg, "DP left: 0x%lx", dl.datapath_id.as_host());
  170
+
  171
+  return CONTINUE;
  172
+}
  173
+
  174
+Disposition JSONStats::handle_ports_stats(const Event& e) {
  175
+  const Port_stats_in_event& ps = assert_cast<const Port_stats_in_event&> (e);
  176
+  datapathid dpid = ps.datapath_id;
  177
+
  178
+  if (streams_[SPRT_PORT_STATS] == NULL) {
  179
+    VLOG_DBG(lg, "Dropping unsolicited ofp_port_stats");
  180
+    return CONTINUE;
  181
+  }
  182
+
  183
+  std::ostringstream& oss = streams_[SPRT_PORT_STATS]->msg;
  184
+  VLOG_DBG(lg, "Handling ports_stats event from 0x%lx", dpid.as_host());
  185
+
  186
+  /* Construct our JSON ofp_port_stats message */
  187
+  if (streams_[SPRT_PORT_STATS]->appending)
  188
+    oss << ",";
  189
+  else
  190
+    streams_[SPRT_PORT_STATS]->appending = 1;
  191
+
  192
+  oss << "{\"datapath_id\":\"" << dpid.as_host() << "\",\"ports\":[";
  193
+  oss.flush();
  194
+
  195
+  std::vector<Port_stats>::const_iterator iter;
  196
+  int appending_port = 0;
  197
+  for (iter = ps.ports.begin(); iter != ps.ports.end(); ++iter) {
  198
+    if (iter->port_no >= OFPP_MAX)
  199
+      continue;
  200
+
  201
+    if (appending_port)
  202
+      oss << ",";
  203
+    else
  204
+      appending_port = 1;
  205
+
  206
+    oss << "{\"port_no\":" << iter->port_no << ",";
  207
+    oss << "\"rx_packets\":\"" << iter->rx_packets << "\",";
  208
+    oss << "\"tx_packets\":\"" << iter->tx_packets << "\",";
  209
+    oss << "\"rx_bytes\":\"" << iter->rx_bytes << "\",";
  210
+    oss << "\"tx_bytes\":\"" << iter->tx_bytes << "\",";
  211
+    oss << "\"rx_dropped\":\"" << iter->rx_dropped << "\",";
  212
+    oss << "\"tx_dropped\":\"" << iter->tx_dropped << "\",";
  213
+    oss << "\"rx_errors\":\"" << iter->rx_errors << "\",";
  214
+    oss << "\"tx_errors\":\"" << iter->tx_errors << "\",";
  215
+    oss << "\"rx_frame_err\":\"" << iter->rx_frame_err << "\",";
  216
+    oss << "\"rx_over_err\":\"" << iter->rx_over_err << "\",";
  217
+    oss << "\"rx_crc_err\":\"" << iter->rx_crc_err << "\",";
  218
+    oss << "\"collisions\":\"" << iter->collisions << "\"}";
  219
+
  220
+    oss.flush();
  221
+  }
  222
+
  223
+  oss << "]}";
  224
+  oss.flush();
  225
+
  226
+  streams_[SPRT_PORT_STATS]->datapaths_left -= 1;
  227
+  if (streams_[SPRT_PORT_STATS]->datapaths_left == 0) {
  228
+    VLOG_DBG(lg, "%s", oss.str().c_str());
  229
+    finalise_stream(SPRT_PORT_STATS);
  230
+  }
  231
+
  232
+  return CONTINUE;
  233
+}
  234
+
  235
+void JSONStats::finalise_stream(int request) {
  236
+  if (streams_[request] != NULL) {
  237
+    streams_[request]->send_to_clients();
  238
+    delete streams_[request];
  239
+    streams_[request] = NULL;
  240
+  }
  241
+}
  242
+
  243
+/**
  244
+ * request_features():
  245
+ *
  246
+ * Fetch ofp_features information from our hashmap and prepare a response
  247
+ */
  248
+void JSONStats::request_features() {
  249
+  if (streams_[SPRT_FEATURES] == NULL ) {
  250
+    VLOG_DBG(lg, "Dropping unsolicited ofp_features_reply");
  251
+    return;
  252
+  }
  253
+
  254
+  std::ostringstream& oss = streams_[SPRT_FEATURES]->msg;
  255
+  VLOG_DBG(lg, "Sending datapath info for %lu datapaths", datapaths_.size());
  256
+
  257
+  int appending_dp = 0;
  258
+  hash_map<datapathid,vector<Port> >::iterator map_iter;
  259
+  for (map_iter = datapaths_.begin(); map_iter != datapaths_.end(); ++map_iter) {
  260
+    if (appending_dp)
  261
+      oss << ",";
  262
+    else
  263
+      appending_dp = 1;
  264
+
  265
+    oss << "{\"datapath_id\":\"" << map_iter->first.as_host() << "\",";
  266
+    /* If we were to make a copy of the entire ofp_switch_features msg that we
  267
+     * recieve from datapaths, then we could encode the below information into
  268
+     * our json replies too. */
  269
+    /*oss << "\"n_buffers\":" << map_iter->second.n_buffers << ",";
  270
+    oss << "\"n_tables\":" << map_iter->second.n_tables << ",";
  271
+    oss << "\"capabilities\":" << map_iter->second.capabilities << ",";
  272
+    oss << "\"actions\":" << map_iter->second.actions << ",";*/
  273
+    oss << "\"ports\":[";
  274
+
  275
+    int appending_port = 0;
  276
+    vector<Port>::iterator port_iter = map_iter->second.begin();
  277
+    for(; port_iter != map_iter->second.end(); ++port_iter) {
  278
+      if (port_iter->port_no >= OFPP_MAX)
  279
+        continue;
  280
+
  281
+      if (appending_port)
  282
+        oss << ",";
  283
+      else
  284
+        appending_port = 1;
  285
+
  286
+      oss << "{\"port_no\":" << port_iter->port_no << ",";
  287
+      oss << "\"hw_addr\":\""<< port_iter->hw_addr << "\",";
  288
+      oss << "\"name\":\""<< port_iter->name.c_str() << "\",";
  289
+      /* speed is made available by vigil::Port, but is not in ofp_phy_port */
  290
+      /*oss << "\"speed\":" << port_iter->speed << ",";*/
  291
+      oss << "\"config\":"<< port_iter->config << ",";
  292
+      oss << "\"state\":"<< port_iter->state << ",";
  293
+      oss << "\"curr\":"<< port_iter->curr << ",";
  294
+      oss << "\"advertised\":"<< port_iter->advertised << ",";
  295
+      oss << "\"supported\":"<< port_iter->supported << ",";
  296
+      oss << "\"peer\":"<< port_iter->peer << "}";
  297
+
  298
+      oss.flush();
  299
+    }
  300
+    oss << "]}";
  301
+    oss.flush();
  302
+
  303
+  }
  304
+
  305
+  VLOG_DBG(lg, "%s", oss.str().c_str());
  306
+
  307
+  /* Set this reply to be handled ASAP -- simplifies other code */
  308
+  timeval tv = { 0, 1 };
  309
+  post(boost::bind(&JSONStats::finalise_stream, this, SPRT_FEATURES), tv);
  310
+}
  311
+
  312
+/**
  313
+ * add_client():
  314
+ *
  315
+ * Adds the given Msg_stream to the list of clients for the given stats
  316
+ * request type. If no request is currently underway, begin a new one.
  317
+ */
  318
+void JSONStats::add_client(stats_pkt_request_type request,
  319
+                               Msg_stream *sock) {
  320
+  /* If there isn't currently a request of this type underway */
  321
+  if (streams_[request] == NULL ) {
  322
+    streams_[request] = new Stats_stream(datapaths_.size(), request);
  323
+
  324
+    switch (request) {
  325
+    case SPRT_FEATURES:
  326
+      request_features();
  327
+      break;
  328
+    case SPRT_PORT_STATS:
  329
+      request_port_stats();
  330
+      break;
  331
+    default:
  332
+      VLOG_ERR(lg, "Unexpected request value in add_client()");
  333
+      break;
  334
+    }
  335
+
  336
+    /**
  337
+     * If we request information from x datapaths and recieve <x replies,
  338
+     * then our stats_stream would never finish its response and send back.
  339
+     * This could be caused by a leaving or unresponsive datapath. Here, we
  340
+     * post a timeout to ensure we will always reply to requests (even if only
  341
+     * with incomplete response data).
  342
+     */
  343
+    timeval tv = { response_timeout_ / 1000, response_timeout_ * 1000 };
  344
+    post(boost::bind(&JSONStats::finalise_stream, this, request), tv);
  345
+  }
  346
+
  347
+  streams_[request]->add(sock);
  348
+}
  349
+
  350
+/**
  351
+ * handle_json_event():
  352
+ *
  353
+ * {
  354
+ *   // We handle json messages with
  355
+ *   "type":"jsonstats",
  356
+ *   // and EITHER
  357
+ *   "command":"features_request"
  358
+ *   // OR
  359
+ *   "command":"port_stats_request"
  360
+ * }
  361
+ */
  362
+Disposition JSONStats::handle_json_event(const Event& e) {
  363
+  const JSONMsg_event& jme = assert_cast<const JSONMsg_event&> (e);
  364
+  json_object *jsonobj = (json_object *)jme.jsonobj.get();
  365
+
  366
+  if (jsonobj->type != json_object::JSONT_DICT)
  367
+    return CONTINUE;
  368
+
  369
+  json_dict *jdict = (json_dict *) jsonobj->object;
  370
+  json_dict::iterator i = jdict->find("type");
  371
+  if (i == jdict->end())
  372
+    return CONTINUE;
  373
+
  374
+  if (i->second->type == json_object::JSONT_STRING &&
  375
+      strncmp(((string *)i->second->object)->c_str(),
  376
+              "jsonstats", strlen("jsonstats")) == 0) {
  377
+    VLOG_DBG(lg, "Handling JSON Event (type %d)\n%s", jsonobj->type,
  378
+             jsonobj->get_string(false).c_str());
  379
+
  380
+    i = jdict->find("command");
  381
+    if (i == jdict->end())
  382
+      return CONTINUE;
  383
+
  384
+    if (i->second->type == json_object::JSONT_STRING) {
  385
+      if (strncmp(((string *)i->second->object)->c_str(),
  386
+          "features_request", strlen("features_request")) == 0) {
  387
+        add_client(SPRT_FEATURES, jme.sock);
  388
+      } else if (strncmp(((string *)i->second->object)->c_str(),
  389
+                 "port_stats_request", strlen("port_stats_request")) == 0) {
  390
+        add_client(SPRT_PORT_STATS, jme.sock);
  391
+      }
  392
+    }
  393
+  }
  394
+
  395
+  return CONTINUE;
  396
+}
  397
+
  398
+REGISTER_COMPONENT(container::Simple_component_factory<JSONStats>, JSONStats);
  399
+
  400
+} /* anonymous namespace */
11  nox/src/nox/netapps/jsonstats/meta.json
... ...
@@ -0,0 +1,11 @@
  1
+{
  2
+    "components": [
  3
+        {
  4
+            "name": "jsonstats",
  5
+            "library": "jsonstats",
  6
+            "dependencies": [
  7
+                "jsonmessenger"
  8
+            ]
  9
+        }
  10
+    ]
  11
+}
53  nox/src/nox/netapps/jsonstats/stats_stream.cc
... ...
@@ -0,0 +1,53 @@
  1
+/*
  2
+ *  Copyright 2012 Fundação CPqD
  3
+ *
  4
+ *   Licensed under the Apache License, Version 2.0 (the "License");
  5
+ *   you may not use this file except in compliance with the License.
  6
+ *   You may obtain a copy of the License at
  7
+ *
  8
+ *      http://www.apache.org/licenses/LICENSE-2.0
  9
+ *
  10
+ *   Unless required by applicable law or agreed to in writing, software
  11
+ *   distributed under the License is distributed on an "AS IS" BASIS,
  12
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13
+ *   See the License for the specific language governing permissions and
  14
+ *   limitations under the License.
  15
+ */
  16
+#include <vector>
  17
+#include <iostream>
  18
+#include <sstream>
  19
+#include "stats_stream.hh"
  20
+
  21
+Stats_stream::Stats_stream(int num_datapaths, stats_pkt_request_type type) {
  22
+  std::string response_string;
  23
+
  24
+  switch(type) {
  25
+    case SPRT_FEATURES:
  26
+      response_string = "features_reply";
  27
+      break;
  28
+    case SPRT_PORT_STATS:
  29
+      response_string = "port_stats";
  30
+      break;
  31
+    default:
  32
+      response_string = "unexpected_type";
  33
+      break;
  34
+  }
  35
+
  36
+  msg << "{\"type\":\"" << response_string << "\",\"datapaths\":[";
  37
+  datapaths_left = num_datapaths;
  38
+  appending = 0;
  39
+}
  40
+
  41
+void Stats_stream::send_to_clients() {
  42
+  msg << "]}\0";
  43
+  msg.flush();
  44
+
  45
+  std::vector<vigil::Msg_stream*>::iterator iter;
  46
+  for(iter = clients_.begin(); iter != clients_.end(); ++iter) {
  47
+    (*iter)->send(msg.str());
  48
+  }
  49
+}
  50
+
  51
+void Stats_stream::add(vigil::Msg_stream *client) {
  52
+  clients_.push_back(client);
  53
+}
39  nox/src/nox/netapps/jsonstats/stats_stream.hh
... ...
@@ -0,0 +1,39 @@
  1
+/*
  2
+ *  Copyright 2012 Fundação CPqD
  3
+ *
  4
+ *   Licensed under the Apache License, Version 2.0 (the "License");
  5
+ *   you may not use this file except in compliance with the License.
  6
+ *   You may obtain a copy of the License at
  7
+ *
  8
+ *      http://www.apache.org/licenses/LICENSE-2.0
  9
+ *
  10
+ *   Unless required by applicable law or agreed to in writing, software
  11
+ *   distributed under the License is distributed on an "AS IS" BASIS,
  12
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13
+ *   See the License for the specific language governing permissions and
  14
+ *   limitations under the License.
  15
+ */
  16
+#ifndef STATS_STREAM_HH
  17
+#define STATS_STREAM_HH
  18
+
  19
+#include "messenger_core.hh"
  20
+
  21
+enum stats_pkt_request_type {
  22
+  SPRT_FEATURES,
  23
+  SPRT_PORT_STATS,
  24
+  SPRT_MAX
  25
+};
  26
+
  27
+class Stats_stream {
  28
+  std::vector<vigil::Msg_stream*> clients_;
  29
+public:
  30
+  int datapaths_left;
  31
+  std::ostringstream msg;
  32
+  int appending;
  33
+
  34
+  Stats_stream(int num_datapaths, stats_pkt_request_type type);
  35
+  void send_to_clients();
  36
+  void add(vigil::Msg_stream *client);
  37
+};
  38
+
  39
+#endif /* STATS_STREAM_HH */

0 notes on commit 20cb3ff

Please sign in to comment.
Something went wrong with that request. Please try again.