Patchwork [BUG:2333,1/3] glusterd: keep mgmt program peerinfo specific

login
register
Submitter Amar Tumballi
Date 2011-02-25 14:05:36
Message ID <20110225140536.GA8325@gluster.com>
Download mbox | patch
Permalink /patch/6268/
State Accepted
Delegated to: Anand Avati
Headers show

Comments

Amar Tumballi - 2011-02-25 14:05:36
With different version of glusterd in different machines, its not
possible to support using just one mgmt program. Instead each
peerinfo structure should have its own mgmt programs, so one glusterd
can support multiple versions

Signed-off-by: Amar Tumballi <amar@gluster.com>
---
 xlators/mgmt/glusterd/src/glusterd-handler.c   |   70 +--------
 xlators/mgmt/glusterd/src/glusterd-handshake.c |  201 +++++++++++++++++++++++-
 xlators/mgmt/glusterd/src/glusterd-sm.h        |    1 +
 xlators/mgmt/glusterd/src/glusterd.c           |    2 -
 xlators/mgmt/glusterd/src/glusterd.h           |    4 +
 5 files changed, 208 insertions(+), 70 deletions(-)

Patch

diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index e9fa043..b4dd5c6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -3657,60 +3657,11 @@  out:
         return ret;
 }
 
-static int
-glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
-{
-        GF_ASSERT (peerctx);
-
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_probe_ctx_t            *ctx = NULL;
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-
-
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_CONNECTED, &event);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get new event");
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
-
-        if (!ctx) {
-                ret = -1;
-                gf_log ("", GF_LOG_ERROR, "Memory not available");
-                goto out;
-        }
-
-        peerinfo = peerctx->peerinfo;
-        ctx->hostname = gf_strdup (peerinfo->hostname);
-        ctx->port = peerinfo->port;
-        ctx->req = peerctx->args.req;
-
-        event->peerinfo = peerinfo;
-        event->ctx = ctx;
-
-        ret = glusterd_friend_sm_inject_event (event);
-
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject "
-                        "EVENT_CONNECTED ret = %d", ret);
-                goto out;
-        }
-
-out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
-}
-
 int
 glusterd_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                      void *data)
 {
         xlator_t                *this = NULL;
-        char                    *handshake = "on";
         glusterd_conf_t         *conf = NULL;
         int                     ret = 0;
         glusterd_peerinfo_t     *peerinfo = NULL;
@@ -3724,30 +3675,15 @@  glusterd_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
         this = THIS;
         conf = this->private;
 
-
         switch (event) {
         case RPC_CLNT_CONNECT:
         {
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
                 peerinfo->connected = 1;
 
-                if ((ret < 0) || (strcasecmp (handshake, "on"))) {
-                        //ret = glusterd_handshake (this, peerinfo->rpc);
-
-                } else {
-                        //conf->rpc->connected = 1;
-                        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
-                }
-
-                if (GD_MODE_ON == peerctx->args.mode) {
-                        ret = glusterd_event_connected_inject (peerctx);
-                        peerctx->args.req = NULL;
-                } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
-                        peerctx->args.mode = GD_MODE_ON;
-                }
-
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
+                ret = glusterd_peer_handshake (this, rpc, peerctx);
+                if (ret)
+                        gf_log ("", GF_LOG_ERROR, "glusterd handshake failed");
                 break;
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index 213bcc6..b8218e7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -24,20 +24,22 @@ 
 #endif
 
 #include "xlator.h"
+#include "defaults.h"
 #include "glusterfs.h"
 #include "compat-errno.h"
 
 #include "glusterd.h"
 #include "glusterd-utils.h"
+#include "glusterd-op-sm.h"
 
 #include "glusterfs3.h"
 #include "protocol-common.h"
 #include "rpcsvc.h"
 
+extern struct rpc_clnt_program glusterd3_1_mgmt_prog;
 
 typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *data);
 
-
 static size_t
 build_volfile_path (const char *volname, char *path,
                     size_t path_len)
@@ -211,3 +213,200 @@  struct rpcsvc_program gluster_handshake_prog = {
         .actors    = gluster_handshake_actors,
         .numactors = GF_HNDSK_MAXVALUE,
 };
+
+char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = {
+        [GF_DUMP_NULL] = "NULL",
+        [GF_DUMP_DUMP] = "DUMP",
+};
+
+rpc_clnt_prog_t glusterd_dump_prog = {
+        .progname  = "GLUSTERD-DUMP",
+        .prognum   = GLUSTER_DUMP_PROGRAM,
+        .progver   = GLUSTER_DUMP_VERSION,
+        .procnames = glusterd_dump_proc,
+};
+
+static int
+glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
+{
+        GF_ASSERT (peerctx);
+
+        glusterd_friend_sm_event_t      *event = NULL;
+        glusterd_probe_ctx_t            *ctx = NULL;
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+
+
+        ret = glusterd_friend_sm_new_event
+                        (GD_FRIEND_EVENT_CONNECTED, &event);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get new event");
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
+
+        if (!ctx) {
+                ret = -1;
+                gf_log ("", GF_LOG_ERROR, "Memory not available");
+                goto out;
+        }
+
+        peerinfo = peerctx->peerinfo;
+        ctx->hostname = gf_strdup (peerinfo->hostname);
+        ctx->port = peerinfo->port;
+        ctx->req = peerctx->args.req;
+
+        event->peerinfo = peerinfo;
+        event->ctx = ctx;
+
+        ret = glusterd_friend_sm_inject_event (event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject "
+                        "EVENT_CONNECTED ret = %d", ret);
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo,
+                                gf_prog_detail *prog)
+{
+        gf_prog_detail *trav     = NULL;
+        int             ret      = -1;
+
+        if (!peerinfo || !prog)
+                goto out;
+
+        trav = prog;
+
+        while (trav) {
+                /* Select 'programs' */
+                if ((glusterd3_1_mgmt_prog.prognum == trav->prognum) &&
+                    (glusterd3_1_mgmt_prog.progver == trav->progver)) {
+                        peerinfo->mgmt = &glusterd3_1_mgmt_prog;
+                        gf_log ("", GF_LOG_INFO,
+                                "Using Program %s, Num (%"PRId64"), "
+                                "Version (%"PRId64")",
+                                trav->progname, trav->prognum, trav->progver);
+                        ret = 0;
+                        break;
+                }
+                if (ret) {
+                        gf_log ("", GF_LOG_TRACE,
+                                "%s (%"PRId64") not supported", trav->progname,
+                                trav->progver);
+                }
+                trav = trav->next;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        int                  ret      = -1;
+        gf_dump_rsp          rsp      = {0,};
+        xlator_t            *this     = NULL;
+        gf_prog_detail      *trav     = NULL;
+        gf_prog_detail      *next     = NULL;
+        call_frame_t        *frame    = NULL;
+        glusterd_peerinfo_t *peerinfo = NULL;
+        glusterd_peerctx_t  *peerctx  = NULL;
+
+        this = THIS;
+        frame = myframe;
+        peerctx = frame->local;
+        peerinfo = peerctx->peerinfo;
+
+        if (-1 == req->rpc_status) {
+                gf_log ("", GF_LOG_ERROR,
+                        "error through RPC layer, retry again later");
+                goto out;
+        }
+
+        ret = xdr_to_dump_rsp (*iov, &rsp);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "failed to decode XDR");
+                goto out;
+        }
+        if (-1 == rsp.op_ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "failed to get the 'versions' from remote server");
+                goto out;
+        }
+
+        /* Make sure we assign the proper program to peer */
+        ret = glusterd_set_clnt_mgmt_program (peerinfo, rsp.prog);
+        if (ret) {
+                gf_log ("", GF_LOG_WARNING, "failed to set the mgmt program");
+                goto out;
+        }
+
+        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
+
+        if (GD_MODE_ON == peerctx->args.mode) {
+                ret = glusterd_event_connected_inject (peerctx);
+                peerctx->args.req = NULL;
+        } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+                peerctx->args.mode = GD_MODE_ON;
+        }
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        ret = 0;
+out:
+
+        /* don't use GF_FREE, buffer was allocated by libc */
+        if (rsp.prog) {
+                trav = rsp.prog;
+                while (trav) {
+                        next = trav->next;
+                        free (trav->progname);
+                        free (trav);
+                        trav = next;
+                }
+        }
+
+        STACK_DESTROY (frame->root);
+
+        if (ret != 0)
+                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+
+        return 0;
+}
+
+
+int
+glusterd_peer_handshake (xlator_t *this, struct rpc_clnt *rpc,
+                         glusterd_peerctx_t *peerctx)
+{
+        call_frame_t        *frame    = NULL;
+        gf_dump_req          req      = {0,};
+        int                  ret      = 0;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        frame->local = peerctx;
+
+        req.gfs_id = 0xcafe;
+
+        ret = glusterd_submit_request (peerctx->peerinfo, &req, frame,
+                                       &glusterd_dump_prog, GF_DUMP_DUMP,
+                                       NULL, xdr_from_dump_req, this,
+                                       glusterd_peer_dump_version_cbk);
+out:
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index b62feb7..b422b49 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -93,6 +93,7 @@  struct glusterd_peerinfo_ {
         struct list_head                uuid_list;
         struct list_head                op_peers_list;
         struct rpc_clnt                 *rpc;
+        rpc_clnt_prog_t                 *mgmt;
         int                             connected;
         glusterd_store_handle_t         *shandle;
         glusterd_sm_tr_log_t            sm_log;
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 478519b..50092f9 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -49,7 +49,6 @@ 
 static uuid_t glusterd_uuid;
 extern struct rpcsvc_program glusterd1_mop_prog;
 extern struct rpcsvc_program gluster_handshake_prog;
-extern struct rpc_clnt_program glusterd3_1_mgmt_prog;
 extern struct rpcsvc_program gluster_pmap_prog;
 extern glusterd_op_info_t opinfo;
 
@@ -373,7 +372,6 @@  init (xlator_t *this)
         INIT_LIST_HEAD (&conf->volumes);
         pthread_mutex_init (&conf->mutex, NULL);
         conf->rpc = rpc;
-        conf->mgmt = &glusterd3_1_mgmt_prog;
         strncpy (conf->workdir, dirname, PATH_MAX);
 
         INIT_LIST_HEAD (&conf->xprt_list);
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index aca7b7d..1103fd5 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -477,5 +477,9 @@  glusterd_volume_txn (rpcsvc_request_t *req, char *volname, int flags,
                      glusterd_op_t op);
 
 int
+glusterd_peer_handshake (xlator_t *this, struct rpc_clnt *rpc,
+                         glusterd_peerctx_t *peerctx);
+
+int
 glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict, char **op_errstr);
 #endif