Aneesh Kumar K.V
2002-03-08 07:41:03 UTC
Hi,
Last patch i sent seem to have some problem .This is the latest patch.
I am right now trying to run it on a two node cluster. I haven't tested
the nodeup and nodedown routine. BTW How do i verify that the DLM is
running properly on the cluster. I am getting proper entry in the
/proc/haDLM and i tried to run the convert_test.sh on both the nodes. It
ran successfully. ( I mean it printed on both the nodes SUCCESS ).
-aneesh
diff -Nru --exclude=test dlm.old/source/Makefile dlm/source/Makefile
--- dlm.old/source/Makefile Wed Dec 5 02:27:47 2001
+++ dlm/source/Makefile Fri Mar 8 12:46:20 2002
@@ -55,7 +55,11 @@
KERNEL_DIRS = dlmdk dlmcccp dlmkernapi
+ifeq ($(CLUSTMGR),ci-linux)
+USER_DIRS = dlmdu libdlm
+else
USER_DIRS = dlmdu dlmdcl libdlm
+endif
.PHONY: clean kernel user kernel_dep user_dep $(KERNEL_DIRS) $(USER_DIRS)
diff -Nru --exclude=test dlm.old/source/dlmdk/Makefile dlm/source/dlmdk/Makefile
--- dlm.old/source/dlmdk/Makefile Sat Nov 3 03:43:55 2001
+++ dlm/source/dlmdk/Makefile Fri Mar 8 12:53:43 2002
@@ -42,21 +42,23 @@
endif
DEFS = -DDBPRINT
-#DEFS += -DDEBUG
+DEFS += -DDEBUG
+KERN_HEADER=/usr/src/linux/include
ifneq ($(SOURCE_PATH),)
VPATH = $(SOURCE_PATH)/dlmdk
-INC = -I. -I$(SOURCE_PATH)/dlmdk -I$(SOURCE_PATH)/include
+INC = -I. -I$(SOURCE_PATH)/dlmdk -I$(SOURCE_PATH)/include -I$(KERN_HEADER)
MKDEP = $(SOURCE_PATH)/mkdep
DLM_DEVMSG_H = $(SOURCE_PATH)/include/dlm_devmsg.h
CLMSTRUCTS_H = $(SOURCE_PATH)/include/clmstructs.h
else
-INC = -I. -I../include
+INC = -I. -I../include -I$(KERN_HEADER)
MKDEP = ../mkdep
DLM_DEVMSG_H = ../include/dlm_devmsg.h
CLMSTRUCTS_H = ../include/clmstructs.h
endif
+
OPT = -O2
DEBUG = -Wall
LIBS =
diff -Nru --exclude=test dlm.old/source/dlmdk/clm_info.c dlm/source/dlmdk/clm_info.c
--- dlm.old/source/dlmdk/clm_info.c Wed Oct 24 03:51:44 2001
+++ dlm/source/dlmdk/clm_info.c Fri Mar 8 14:21:55 2002
@@ -652,6 +652,7 @@
#endif
node_count = clm_node_count();
+
#ifdef DEBUG
count = node_count;
#endif
@@ -666,6 +667,7 @@
i = (i + 1) % node_count;
node_id = INDEX_TO_NODEID( i );
+
#ifdef DEBUG
CLM_ASSERT( count != 0, "Too many loop iterations!" );
count--;
diff -Nru --exclude=test dlm.old/source/dlmdk/dlm_kerndd.c dlm/source/dlmdk/dlm_kerndd.c
--- dlm.old/source/dlmdk/dlm_kerndd.c Sat Oct 20 04:12:38 2001
+++ dlm/source/dlmdk/dlm_kerndd.c Fri Mar 8 14:24:31 2002
@@ -71,6 +71,12 @@
* proc_print_vdevice
*/
+/* 8 MARCH 2002 ( ***@digital.com)
+ * Added support for kernel cluster manager
+ *define KERN_CLMGR in the user space and
+ * and CONFIG_CLMS in the kernel space for Cluster Infrastructure for Linux
+ */
+
/* ============================================================ */
#ifndef __KERNEL__
#define __KERNEL__
@@ -100,6 +106,14 @@
#include <linux/timer.h>
#include <asm/uaccess.h>
+
+#ifdef CONFIG_CLMS
+#include <cluster/clms.h>
+/* To avoid conflicting type with nsc.h and dlm header files*/
+#define BOOL_T_DEFINED
+#endif
+
+
#include "dlm.h"
#include "ast.h"
#include "ast_stub.h"
@@ -112,6 +126,9 @@
#include "dlm_devmsg.h"
#include "dlm_workqueue.h"
#include "cltrc.h"
+#ifdef CONFIG_CLMS
+#include "dlm_clust.h"
+#endif
#include "dlm_proto.h" /* Should be LAST included file. */
@@ -131,6 +148,11 @@
int haDLM_admin_minor = 0; /* Minor number for "admin" device */
int haDLM_locks_minor = 0; /* Minor number for "locks" device */
+#ifdef CONFIG_CLMS
+ static MQ_DLM_TOP_INFO_t *cli_topo;
+ static int first_clmgr_call = 1 ;
+#endif
+
MODULE_AUTHOR("Peter Wombat");
MODULE_DESCRIPTION("The ha Distributed Lock Manager device.");
MODULE_PARM(haDLM_major_number, "i"); /* an int */
@@ -164,6 +186,22 @@
struct file *file,
unsigned int cmd,
unsigned long arg);
+#ifdef CONFIG_CLMS
+
+int dlm_cli_nodeup( void *clms_handle,
+ int service,
+ clusternode_t node,
+ clusternode_t surrogate,
+ void *private) ;
+int dlm_cli_nodedown( void *clms_handle,
+ int service,
+ clusternode_t node,
+ clusternode_t surrogate,
+ void *private) ;
+
+MQ_DLM_TOP_INFO_t * get_upnode_list(void);
+#endif
+
/* ------------------------------------------------------------ */
/*
@@ -231,6 +269,7 @@
static unsigned long jiffies_per_second;
+
/* ------------------------------------------------------------ */
/*
* Keep track of admin client.
@@ -328,6 +367,7 @@
{
int _i;
+
if ( haDLM_name == NULL ) {
bsdlog( LOG_ERR, "haDLM_name not set by module loader. Terminating.\n" );
return( -1 );
@@ -363,12 +403,24 @@
dlm_proc_start( haDLM_name );
#endif /* CONFIG_PROC_FS */
- printk("[%s] Success, register device [%d/%s]\n",
- haDLM_name,
- haDLM_major_number,
- haDLM_name);
- return(0);
+#ifdef CONFIG_CLMS
+ return (register_clms_subsys("dlm",
+ -1,
+ dlm_cli_nodeup,
+ dlm_cli_nodedown,
+ NULL,
+ NULL,
+ NULL));
+
+#endif
+
+ printk("[%s] Success, register device [%d/%s]\n",
+ haDLM_name,
+ haDLM_major_number,
+ haDLM_name);
+
+ return(0);
}
/* ------------------------------------------------------------ */
@@ -792,8 +844,10 @@
dlm_MSG_t *message;
dlm_MSG_types_t command;
MQ_DLM_TOP_INFO_t *topo;
+
dlm_workunit_t *wu;
+
_rc = lbuf;
_device = (struct our_vdevice *)filp->private_data;
@@ -868,7 +922,9 @@
haDLM_name,
kproc_pid);
}
+#ifndef CONFIG_CLMS
haDLM_allow_locks = 1;
+#endif
break;
case haDLM_MSG_Stop:
printk("[%s] STOP message received\n",
@@ -899,6 +955,22 @@
haDLM_name,
lbuf);
}
+
+#ifdef CONFIG_CLMS
+ /*check here if the call is the first call. If so use this for
+ * building the array of nodes in which dlm is running
+ */
+ if(first_clmgr_call)
+ {
+
+ cli_topo = topo;
+ first_clmgr_call = 0;
+ /* Only after building the initial topology i should accept locks */
+ haDLM_allow_locks = 1;
+ /* Preparing the new topology with node up information */
+ topo = get_upnode_list();
+ }
+#endif
if (NULL ==
(wu = kmalloc_something(sizeof(dlm_workunit_t),
"WorkUnit for topology block"))) {
@@ -1337,3 +1409,98 @@
return(_rc);
}
+
+#ifdef CONFIG_CLMS
+
+int dlm_cli_nodeup( void *clms_handle,
+ int service,
+ clusternode_t node,
+ clusternode_t surrogate,
+ void *private)
+{
+ MQ_DLM_TOP_INFO_t *new_topo;
+ dlm_workunit_t *wu;
+
+ if (!haDLM_allow_locks) {
+ printk("locks: not yet initialized.\n");
+ clms_nodeup_callback(clms_handle, service, node);
+ return 0;
+ }
+
+ new_topo = get_upnode_list();
+ if (new_topo == NULL)
+ return(-ENOMEM); /* Check for the return */
+ if (NULL ==
+ (wu = kmalloc_something(sizeof(dlm_workunit_t),
+ "WorkUnit for topology block"))) {
+ return(-ENOMEM);
+ }
+ wu->data = new_topo;
+ wu->free_data = kfree_topo_block;
+ wu->type = MQ_DLM_TOP_INFO_MSG;
+ dlm_workqueue_put_work(dlm_master_work_queue, wu);
+ clms_nodeup_callback(clms_handle, service, node);
+ return 0;
+}
+
+
+int dlm_cli_nodedown( void *clms_handle,
+ int service,
+ clusternode_t node,
+ clusternode_t surrogate,
+ void *private)
+{
+ MQ_DLM_TOP_INFO_t *new_topo;
+ dlm_workunit_t *wu;
+
+ if (!haDLM_allow_locks) {
+ printk("locks: not yet initialized.\n");
+ clms_nodedown_callback(clms_handle, service, node);
+ return 0;
+ }
+
+
+ new_topo = get_upnode_list();
+ if (new_topo == NULL)
+ return(-ENOMEM);
+ if (NULL ==
+ (wu = kmalloc_something(sizeof(dlm_workunit_t),
+ "WorkUnit for topology block"))) {
+ return(-ENOMEM);
+ }
+ wu->data = new_topo;
+ wu->free_data = kfree_topo_block;
+ wu->type = MQ_DLM_TOP_INFO_MSG;
+ dlm_workqueue_put_work(dlm_master_work_queue, wu);
+ clms_nodedown_callback(clms_handle, service, node);
+ return 0;
+}
+/* Getting the list of already up nodes */
+MQ_DLM_TOP_INFO_t * get_upnode_list(void)
+{
+ MQ_DLM_TOP_INFO_t *new_topo;
+ MQ_DLM_USEADDR_t *addr;
+ int i ;
+ int size = (cli_topo->n_nodes * sizeof(MQ_DLM_USEADDR_t)) + sizeof(MQ_DLM_TOP_INFO_t);
+
+ if (NULL ==
+ (new_topo = kmalloc_something(size, "CMGR topology block"))) {
+ return(NULL);
+ }
+
+ memcpy(new_topo,cli_topo,size);
+
+ for(i= 0 ; i <new_topo->n_nodes;i++ ) {
+ addr = &(new_topo->addrs[i]);
+ if(!clms_isnodeup((clusternode_t)addr->nodeid)) {
+ addr->dlm_major = 0;
+ addr->dlm_minor = 0;
+ addr->useaddr.s_addr = 0;
+ }
+ }
+
+ return new_topo;
+
+ }
+
+#endif /* CONFIG_CLMS */
diff -Nru --exclude=test dlm.old/source/dlmdu/Makefile dlm/source/dlmdu/Makefile
--- dlm.old/source/dlmdu/Makefile Fri Mar 9 07:05:16 2001
+++ dlm/source/dlmdu/Makefile Fri Mar 8 14:11:28 2002
@@ -78,13 +78,19 @@
# which one (either rsct or heartbeat.)
+ifeq ($(CLUSTMGR),ci-linux)
+DEFS += -DKERN_CLMGR
+LIBS = -lpthread -ldl -lcluster
+else
ifneq ($(CLUSTMGR),)
DEFS += -DDIRECT_CLUSTINT
ifeq ($(CLUSTMGR),rsct)
LIBS = -lha_gs_r -lpthread
else
+ifeq ($(CLUSTMGR),hearbeat)
LIBS = -lhbclient -lpthread
+endif
endif
else
@@ -93,6 +99,7 @@
LIBS = -lpthread -ldl
endif
+endif
CPPFLAGS += $(DEFS) $(INC)
PROTOFLAGS = -e -DCPROTO $(CPPFLAGS)
@@ -109,6 +116,7 @@
daemon_sources += dlm_heartbeat.c
endif
+
OFILES_U= $(daemon_sources:.c=.o)
CFILES = $(daemon_sources)
diff -Nru --exclude=test dlm.old/source/dlmdu/dlm_daemon.c dlm/source/dlmdu/dlm_daemon.c
--- dlm.old/source/dlmdu/dlm_daemon.c Thu Oct 25 00:03:03 2001
+++ dlm/source/dlmdu/dlm_daemon.c Fri Mar 8 15:14:46 2002
@@ -124,9 +124,11 @@
#include <time.h>
#include <assert.h>
+#ifndef KERN_CLMGR
#ifndef DIRECT_CLUSTINT
#include <dlfcn.h> /* Dynamic load support. */
#endif /* !DIRECT_CLUSTINT */
+#endif /* KERN_CLMGR */
#include "dlm.h"
#include "dlm_devmsg.h"
@@ -176,10 +178,15 @@
* Functions in this mainline manual that are used to interact
* with the cluster interface code.
*/
+#ifndef KERN_CLMGR
+
int cluster_manager_setup(dlm_node_array_t *nodes);
#ifndef DIRECT_CLUSTINT
void *cluster_interface_call(void *arg);
#endif /* !DIRECT_CLUSTINT */
+
+#endif /* KERN_CLMGR */
+
int drain_event_queue(MQ_DLM_TOP_INFO_t *node_events);
int find_configured_node(dlm_node_t *node);
@@ -246,17 +253,21 @@
admin_fd = -1;
}
+#ifndef KERN_CLMGR
+
if (0 != cmgr_thread_id) {
#ifdef DIRECT_CLUSTINT
cmgr_thread_id = 0;
fprintf(stderr,
"Shutting down cluster interface.\n");
- cluster_interface_shutdown();
+ cluster_interface_shutdown(); /* do I need to say need_to_quit here*/
#else /* DIRECT_CLUSTINT */
(*cluster_interface_stop)();
#endif /* DIRECT_CLUSTINT */
}
+#endif /* KERN_CLMGR */
+
#ifdef _ON_AIX_DLM_
mid_t kmid;
int status;
@@ -501,12 +512,14 @@
* work. Once we are connected, start up the kernel
* thread so that it is ready to work.
*/
+#ifndef KERN_CLMGR
if (-1 == cluster_manager_setup(configured_nodes)) {
fprintf(stderr,
"Error loading cluster interface module. Exiting.\n");
cleanup();
exit(5);
}
+#endif
message.msg_code = haDLM_MSG_Init;
message.msg_ptr_size = sizeof(args);
@@ -557,9 +570,69 @@
/* initialize message buffer; reused for each updata
*/
initialize_node_events(node_events);
+#ifdef KERN_CLMGR
+ message.msg_code = haDLM_MSG_CMGR;
+ message.msg_ptr_size = msize;
+ message.topo_array = node_events;
- while(!need_to_quit) {
+ j = initial_topology(node_events);
+
+ if (0 < j) {
+
+ rc = pthread_mutex_unlock(¬ify_mutex);
+ if (0 != rc) {
+ fprintf(stderr,
+ "Cannot unlock notify_mutex in main loop, rc = [%d]\n",
+ rc);
+ cleanup();
+ exit(rc);
+ }
+
+ i = write(admin_fd, &message, sizeof(message)); /* Tell kernel. */
+ if (i != sizeof(message)) {
+ fprintf(stderr,
+ "Error on write, i/err[%d/%s], code/size[%d/%d]\n",
+ i,
+ strerror(errno),
+ message.msg_code,
+ sizeof(message));
+ cleanup();
+ assert(!"Error writing through DLM admin device");
+ } else {
+ admin_calls++;
+
+ fprintf(stderr,
+ "Wrote code/size[%d/%d], blocks so far [%d]\n",
+ message.msg_code,
+ sizeof(message),
+ admin_calls);
+ }
+ rc = pthread_mutex_lock(¬ify_mutex);
+ if (0 != rc) {
+ fprintf(stderr,
+ "Cannot lock notify_mutex in main loop, rc = [%d]\n",
+ rc);
+ cleanup();
+ exit(rc);
+ }
+ time_since = 0;
+
+ }
+ else {
+ fprintf(stderr, "Cannot find the initial topology\n");
+ cleanup();
+ exit(rc);
+
+ }
+#endif
+
+ while(!need_to_quit) {
+/* Since node monitoring is now done inside the kernel there is no need
+ * for a timedout node updation from here. So just looping to make sure.
+ * dlmdu runs throught out. ( This may not be acutally needed )
+ */
+#ifndef KERN_CLMGR /* I am not sure whether this should be the one */
if (0 < nodeq_count) {
/*
* Work to do! Node block(s) waiting.
@@ -626,6 +699,7 @@
"No work to do in handling event queue.\n");
}
+
/*
* Mutex is held at this point, regardless of path above.
*/
@@ -659,6 +733,7 @@
* if we timed out. If there are none, we'll just get right
* back to the conditon wait anyway.
*/
+#endif
} /* while(!need_to_quit) */
@@ -974,6 +1049,8 @@
return(found);
}
+#ifndef KERN_CLMGR
+
/* ============================================================ */
/*
* Set up to call the cluster manager.
@@ -1105,6 +1182,7 @@
}
#endif /* !DIRECT_CLUSTINT */
+#endif /* KERN_CLMGR */
/* ------------------------------------------------------------ */
/*
* With a view to supporting dynamic reconfiguration of the
@@ -1250,3 +1328,30 @@
return(ret);
}
+
+#ifdef KERN_CLMGR
+
+int
+initial_topology( MQ_DLM_TOP_INFO_t *node_events )
+{
+ int i ;
+ node_events->n_nodes = configured_nodes->dlm_node_count;
+ node_events->msg_version = TAM_version;
+ this_node_number = clusternode_num();
+ node_events->this_nodeid = this_node_number;
+ node_events->event = admin_calls; /* Check this ? */
+
+ for( i = 0 ; i < configured_nodes->dlm_node_count; i++)
+ {
+ node_events->addrs[i].nodeid =configured_nodes->dlm_node_entries[i].dlm_node_number;
+ node_events->addrs[i].useaddr.s_addr =configured_nodes->dlm_node_entries[i].dlm_node_address.s_addr;
+ node_events->addrs[i].dlm_major= DLM_MAJOR_VERSION;
+ node_events->addrs[i].dlm_minor= DLM_MINOR_VERSION;
+
+
+ }
+
+printf("Returing the intial topology no of nodes=%d\m",configured_nodes->dlm_node_count);
+ return(configured_nodes->dlm_node_count );
+}
+#endif
diff -Nru --exclude=test dlm.old/source/include/dlm_kernel.h dlm/source/include/dlm_kernel.h
--- dlm.old/source/include/dlm_kernel.h Thu Nov 8 05:27:09 2001
+++ dlm/source/include/dlm_kernel.h Fri Mar 8 12:58:30 2002
@@ -234,12 +234,15 @@
struct timeval *t1,
struct timeval *t2);
+#ifndef LINUX_VERSION_CODE
+#include <linux/version.h>
#if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,10)
# define snprintf(buf,count,fmt, args...) \
sprintf( buf, fmt, ## args )
# define vsnprintf(buf,count,fmt,args) \
vsprintf( buf, fmt, args )
#endif
+#endif
/*
* Fake float stuff.
Last patch i sent seem to have some problem .This is the latest patch.
I am right now trying to run it on a two node cluster. I haven't tested
the nodeup and nodedown routine. BTW How do i verify that the DLM is
running properly on the cluster. I am getting proper entry in the
/proc/haDLM and i tried to run the convert_test.sh on both the nodes. It
ran successfully. ( I mean it printed on both the nodes SUCCESS ).
-aneesh
diff -Nru --exclude=test dlm.old/source/Makefile dlm/source/Makefile
--- dlm.old/source/Makefile Wed Dec 5 02:27:47 2001
+++ dlm/source/Makefile Fri Mar 8 12:46:20 2002
@@ -55,7 +55,11 @@
KERNEL_DIRS = dlmdk dlmcccp dlmkernapi
+ifeq ($(CLUSTMGR),ci-linux)
+USER_DIRS = dlmdu libdlm
+else
USER_DIRS = dlmdu dlmdcl libdlm
+endif
.PHONY: clean kernel user kernel_dep user_dep $(KERNEL_DIRS) $(USER_DIRS)
diff -Nru --exclude=test dlm.old/source/dlmdk/Makefile dlm/source/dlmdk/Makefile
--- dlm.old/source/dlmdk/Makefile Sat Nov 3 03:43:55 2001
+++ dlm/source/dlmdk/Makefile Fri Mar 8 12:53:43 2002
@@ -42,21 +42,23 @@
endif
DEFS = -DDBPRINT
-#DEFS += -DDEBUG
+DEFS += -DDEBUG
+KERN_HEADER=/usr/src/linux/include
ifneq ($(SOURCE_PATH),)
VPATH = $(SOURCE_PATH)/dlmdk
-INC = -I. -I$(SOURCE_PATH)/dlmdk -I$(SOURCE_PATH)/include
+INC = -I. -I$(SOURCE_PATH)/dlmdk -I$(SOURCE_PATH)/include -I$(KERN_HEADER)
MKDEP = $(SOURCE_PATH)/mkdep
DLM_DEVMSG_H = $(SOURCE_PATH)/include/dlm_devmsg.h
CLMSTRUCTS_H = $(SOURCE_PATH)/include/clmstructs.h
else
-INC = -I. -I../include
+INC = -I. -I../include -I$(KERN_HEADER)
MKDEP = ../mkdep
DLM_DEVMSG_H = ../include/dlm_devmsg.h
CLMSTRUCTS_H = ../include/clmstructs.h
endif
+
OPT = -O2
DEBUG = -Wall
LIBS =
diff -Nru --exclude=test dlm.old/source/dlmdk/clm_info.c dlm/source/dlmdk/clm_info.c
--- dlm.old/source/dlmdk/clm_info.c Wed Oct 24 03:51:44 2001
+++ dlm/source/dlmdk/clm_info.c Fri Mar 8 14:21:55 2002
@@ -652,6 +652,7 @@
#endif
node_count = clm_node_count();
+
#ifdef DEBUG
count = node_count;
#endif
@@ -666,6 +667,7 @@
i = (i + 1) % node_count;
node_id = INDEX_TO_NODEID( i );
+
#ifdef DEBUG
CLM_ASSERT( count != 0, "Too many loop iterations!" );
count--;
diff -Nru --exclude=test dlm.old/source/dlmdk/dlm_kerndd.c dlm/source/dlmdk/dlm_kerndd.c
--- dlm.old/source/dlmdk/dlm_kerndd.c Sat Oct 20 04:12:38 2001
+++ dlm/source/dlmdk/dlm_kerndd.c Fri Mar 8 14:24:31 2002
@@ -71,6 +71,12 @@
* proc_print_vdevice
*/
+/* 8 MARCH 2002 ( ***@digital.com)
+ * Added support for kernel cluster manager
+ *define KERN_CLMGR in the user space and
+ * and CONFIG_CLMS in the kernel space for Cluster Infrastructure for Linux
+ */
+
/* ============================================================ */
#ifndef __KERNEL__
#define __KERNEL__
@@ -100,6 +106,14 @@
#include <linux/timer.h>
#include <asm/uaccess.h>
+
+#ifdef CONFIG_CLMS
+#include <cluster/clms.h>
+/* To avoid conflicting type with nsc.h and dlm header files*/
+#define BOOL_T_DEFINED
+#endif
+
+
#include "dlm.h"
#include "ast.h"
#include "ast_stub.h"
@@ -112,6 +126,9 @@
#include "dlm_devmsg.h"
#include "dlm_workqueue.h"
#include "cltrc.h"
+#ifdef CONFIG_CLMS
+#include "dlm_clust.h"
+#endif
#include "dlm_proto.h" /* Should be LAST included file. */
@@ -131,6 +148,11 @@
int haDLM_admin_minor = 0; /* Minor number for "admin" device */
int haDLM_locks_minor = 0; /* Minor number for "locks" device */
+#ifdef CONFIG_CLMS
+ static MQ_DLM_TOP_INFO_t *cli_topo;
+ static int first_clmgr_call = 1 ;
+#endif
+
MODULE_AUTHOR("Peter Wombat");
MODULE_DESCRIPTION("The ha Distributed Lock Manager device.");
MODULE_PARM(haDLM_major_number, "i"); /* an int */
@@ -164,6 +186,22 @@
struct file *file,
unsigned int cmd,
unsigned long arg);
+#ifdef CONFIG_CLMS
+
+int dlm_cli_nodeup( void *clms_handle,
+ int service,
+ clusternode_t node,
+ clusternode_t surrogate,
+ void *private) ;
+int dlm_cli_nodedown( void *clms_handle,
+ int service,
+ clusternode_t node,
+ clusternode_t surrogate,
+ void *private) ;
+
+MQ_DLM_TOP_INFO_t * get_upnode_list(void);
+#endif
+
/* ------------------------------------------------------------ */
/*
@@ -231,6 +269,7 @@
static unsigned long jiffies_per_second;
+
/* ------------------------------------------------------------ */
/*
* Keep track of admin client.
@@ -328,6 +367,7 @@
{
int _i;
+
if ( haDLM_name == NULL ) {
bsdlog( LOG_ERR, "haDLM_name not set by module loader. Terminating.\n" );
return( -1 );
@@ -363,12 +403,24 @@
dlm_proc_start( haDLM_name );
#endif /* CONFIG_PROC_FS */
- printk("[%s] Success, register device [%d/%s]\n",
- haDLM_name,
- haDLM_major_number,
- haDLM_name);
- return(0);
+#ifdef CONFIG_CLMS
+ return (register_clms_subsys("dlm",
+ -1,
+ dlm_cli_nodeup,
+ dlm_cli_nodedown,
+ NULL,
+ NULL,
+ NULL));
+
+#endif
+
+ printk("[%s] Success, register device [%d/%s]\n",
+ haDLM_name,
+ haDLM_major_number,
+ haDLM_name);
+
+ return(0);
}
/* ------------------------------------------------------------ */
@@ -792,8 +844,10 @@
dlm_MSG_t *message;
dlm_MSG_types_t command;
MQ_DLM_TOP_INFO_t *topo;
+
dlm_workunit_t *wu;
+
_rc = lbuf;
_device = (struct our_vdevice *)filp->private_data;
@@ -868,7 +922,9 @@
haDLM_name,
kproc_pid);
}
+#ifndef CONFIG_CLMS
haDLM_allow_locks = 1;
+#endif
break;
case haDLM_MSG_Stop:
printk("[%s] STOP message received\n",
@@ -899,6 +955,22 @@
haDLM_name,
lbuf);
}
+
+#ifdef CONFIG_CLMS
+ /*check here if the call is the first call. If so use this for
+ * building the array of nodes in which dlm is running
+ */
+ if(first_clmgr_call)
+ {
+
+ cli_topo = topo;
+ first_clmgr_call = 0;
+ /* Only after building the initial topology i should accept locks */
+ haDLM_allow_locks = 1;
+ /* Preparing the new topology with node up information */
+ topo = get_upnode_list();
+ }
+#endif
if (NULL ==
(wu = kmalloc_something(sizeof(dlm_workunit_t),
"WorkUnit for topology block"))) {
@@ -1337,3 +1409,98 @@
return(_rc);
}
+
+#ifdef CONFIG_CLMS
+
+int dlm_cli_nodeup( void *clms_handle,
+ int service,
+ clusternode_t node,
+ clusternode_t surrogate,
+ void *private)
+{
+ MQ_DLM_TOP_INFO_t *new_topo;
+ dlm_workunit_t *wu;
+
+ if (!haDLM_allow_locks) {
+ printk("locks: not yet initialized.\n");
+ clms_nodeup_callback(clms_handle, service, node);
+ return 0;
+ }
+
+ new_topo = get_upnode_list();
+ if (new_topo == NULL)
+ return(-ENOMEM); /* Check for the return */
+ if (NULL ==
+ (wu = kmalloc_something(sizeof(dlm_workunit_t),
+ "WorkUnit for topology block"))) {
+ return(-ENOMEM);
+ }
+ wu->data = new_topo;
+ wu->free_data = kfree_topo_block;
+ wu->type = MQ_DLM_TOP_INFO_MSG;
+ dlm_workqueue_put_work(dlm_master_work_queue, wu);
+ clms_nodeup_callback(clms_handle, service, node);
+ return 0;
+}
+
+
+int dlm_cli_nodedown( void *clms_handle,
+ int service,
+ clusternode_t node,
+ clusternode_t surrogate,
+ void *private)
+{
+ MQ_DLM_TOP_INFO_t *new_topo;
+ dlm_workunit_t *wu;
+
+ if (!haDLM_allow_locks) {
+ printk("locks: not yet initialized.\n");
+ clms_nodedown_callback(clms_handle, service, node);
+ return 0;
+ }
+
+
+ new_topo = get_upnode_list();
+ if (new_topo == NULL)
+ return(-ENOMEM);
+ if (NULL ==
+ (wu = kmalloc_something(sizeof(dlm_workunit_t),
+ "WorkUnit for topology block"))) {
+ return(-ENOMEM);
+ }
+ wu->data = new_topo;
+ wu->free_data = kfree_topo_block;
+ wu->type = MQ_DLM_TOP_INFO_MSG;
+ dlm_workqueue_put_work(dlm_master_work_queue, wu);
+ clms_nodedown_callback(clms_handle, service, node);
+ return 0;
+}
+/* Getting the list of already up nodes */
+MQ_DLM_TOP_INFO_t * get_upnode_list(void)
+{
+ MQ_DLM_TOP_INFO_t *new_topo;
+ MQ_DLM_USEADDR_t *addr;
+ int i ;
+ int size = (cli_topo->n_nodes * sizeof(MQ_DLM_USEADDR_t)) + sizeof(MQ_DLM_TOP_INFO_t);
+
+ if (NULL ==
+ (new_topo = kmalloc_something(size, "CMGR topology block"))) {
+ return(NULL);
+ }
+
+ memcpy(new_topo,cli_topo,size);
+
+ for(i= 0 ; i <new_topo->n_nodes;i++ ) {
+ addr = &(new_topo->addrs[i]);
+ if(!clms_isnodeup((clusternode_t)addr->nodeid)) {
+ addr->dlm_major = 0;
+ addr->dlm_minor = 0;
+ addr->useaddr.s_addr = 0;
+ }
+ }
+
+ return new_topo;
+
+ }
+
+#endif /* CONFIG_CLMS */
diff -Nru --exclude=test dlm.old/source/dlmdu/Makefile dlm/source/dlmdu/Makefile
--- dlm.old/source/dlmdu/Makefile Fri Mar 9 07:05:16 2001
+++ dlm/source/dlmdu/Makefile Fri Mar 8 14:11:28 2002
@@ -78,13 +78,19 @@
# which one (either rsct or heartbeat.)
+ifeq ($(CLUSTMGR),ci-linux)
+DEFS += -DKERN_CLMGR
+LIBS = -lpthread -ldl -lcluster
+else
ifneq ($(CLUSTMGR),)
DEFS += -DDIRECT_CLUSTINT
ifeq ($(CLUSTMGR),rsct)
LIBS = -lha_gs_r -lpthread
else
+ifeq ($(CLUSTMGR),hearbeat)
LIBS = -lhbclient -lpthread
+endif
endif
else
@@ -93,6 +99,7 @@
LIBS = -lpthread -ldl
endif
+endif
CPPFLAGS += $(DEFS) $(INC)
PROTOFLAGS = -e -DCPROTO $(CPPFLAGS)
@@ -109,6 +116,7 @@
daemon_sources += dlm_heartbeat.c
endif
+
OFILES_U= $(daemon_sources:.c=.o)
CFILES = $(daemon_sources)
diff -Nru --exclude=test dlm.old/source/dlmdu/dlm_daemon.c dlm/source/dlmdu/dlm_daemon.c
--- dlm.old/source/dlmdu/dlm_daemon.c Thu Oct 25 00:03:03 2001
+++ dlm/source/dlmdu/dlm_daemon.c Fri Mar 8 15:14:46 2002
@@ -124,9 +124,11 @@
#include <time.h>
#include <assert.h>
+#ifndef KERN_CLMGR
#ifndef DIRECT_CLUSTINT
#include <dlfcn.h> /* Dynamic load support. */
#endif /* !DIRECT_CLUSTINT */
+#endif /* KERN_CLMGR */
#include "dlm.h"
#include "dlm_devmsg.h"
@@ -176,10 +178,15 @@
* Functions in this mainline manual that are used to interact
* with the cluster interface code.
*/
+#ifndef KERN_CLMGR
+
int cluster_manager_setup(dlm_node_array_t *nodes);
#ifndef DIRECT_CLUSTINT
void *cluster_interface_call(void *arg);
#endif /* !DIRECT_CLUSTINT */
+
+#endif /* KERN_CLMGR */
+
int drain_event_queue(MQ_DLM_TOP_INFO_t *node_events);
int find_configured_node(dlm_node_t *node);
@@ -246,17 +253,21 @@
admin_fd = -1;
}
+#ifndef KERN_CLMGR
+
if (0 != cmgr_thread_id) {
#ifdef DIRECT_CLUSTINT
cmgr_thread_id = 0;
fprintf(stderr,
"Shutting down cluster interface.\n");
- cluster_interface_shutdown();
+ cluster_interface_shutdown(); /* do I need to say need_to_quit here*/
#else /* DIRECT_CLUSTINT */
(*cluster_interface_stop)();
#endif /* DIRECT_CLUSTINT */
}
+#endif /* KERN_CLMGR */
+
#ifdef _ON_AIX_DLM_
mid_t kmid;
int status;
@@ -501,12 +512,14 @@
* work. Once we are connected, start up the kernel
* thread so that it is ready to work.
*/
+#ifndef KERN_CLMGR
if (-1 == cluster_manager_setup(configured_nodes)) {
fprintf(stderr,
"Error loading cluster interface module. Exiting.\n");
cleanup();
exit(5);
}
+#endif
message.msg_code = haDLM_MSG_Init;
message.msg_ptr_size = sizeof(args);
@@ -557,9 +570,69 @@
/* initialize message buffer; reused for each updata
*/
initialize_node_events(node_events);
+#ifdef KERN_CLMGR
+ message.msg_code = haDLM_MSG_CMGR;
+ message.msg_ptr_size = msize;
+ message.topo_array = node_events;
- while(!need_to_quit) {
+ j = initial_topology(node_events);
+
+ if (0 < j) {
+
+ rc = pthread_mutex_unlock(¬ify_mutex);
+ if (0 != rc) {
+ fprintf(stderr,
+ "Cannot unlock notify_mutex in main loop, rc = [%d]\n",
+ rc);
+ cleanup();
+ exit(rc);
+ }
+
+ i = write(admin_fd, &message, sizeof(message)); /* Tell kernel. */
+ if (i != sizeof(message)) {
+ fprintf(stderr,
+ "Error on write, i/err[%d/%s], code/size[%d/%d]\n",
+ i,
+ strerror(errno),
+ message.msg_code,
+ sizeof(message));
+ cleanup();
+ assert(!"Error writing through DLM admin device");
+ } else {
+ admin_calls++;
+
+ fprintf(stderr,
+ "Wrote code/size[%d/%d], blocks so far [%d]\n",
+ message.msg_code,
+ sizeof(message),
+ admin_calls);
+ }
+ rc = pthread_mutex_lock(¬ify_mutex);
+ if (0 != rc) {
+ fprintf(stderr,
+ "Cannot lock notify_mutex in main loop, rc = [%d]\n",
+ rc);
+ cleanup();
+ exit(rc);
+ }
+ time_since = 0;
+
+ }
+ else {
+ fprintf(stderr, "Cannot find the initial topology\n");
+ cleanup();
+ exit(rc);
+
+ }
+#endif
+
+ while(!need_to_quit) {
+/* Since node monitoring is now done inside the kernel there is no need
+ * for a timedout node updation from here. So just looping to make sure.
+ * dlmdu runs throught out. ( This may not be acutally needed )
+ */
+#ifndef KERN_CLMGR /* I am not sure whether this should be the one */
if (0 < nodeq_count) {
/*
* Work to do! Node block(s) waiting.
@@ -626,6 +699,7 @@
"No work to do in handling event queue.\n");
}
+
/*
* Mutex is held at this point, regardless of path above.
*/
@@ -659,6 +733,7 @@
* if we timed out. If there are none, we'll just get right
* back to the conditon wait anyway.
*/
+#endif
} /* while(!need_to_quit) */
@@ -974,6 +1049,8 @@
return(found);
}
+#ifndef KERN_CLMGR
+
/* ============================================================ */
/*
* Set up to call the cluster manager.
@@ -1105,6 +1182,7 @@
}
#endif /* !DIRECT_CLUSTINT */
+#endif /* KERN_CLMGR */
/* ------------------------------------------------------------ */
/*
* With a view to supporting dynamic reconfiguration of the
@@ -1250,3 +1328,30 @@
return(ret);
}
+
+#ifdef KERN_CLMGR
+
+int
+initial_topology( MQ_DLM_TOP_INFO_t *node_events )
+{
+ int i ;
+ node_events->n_nodes = configured_nodes->dlm_node_count;
+ node_events->msg_version = TAM_version;
+ this_node_number = clusternode_num();
+ node_events->this_nodeid = this_node_number;
+ node_events->event = admin_calls; /* Check this ? */
+
+ for( i = 0 ; i < configured_nodes->dlm_node_count; i++)
+ {
+ node_events->addrs[i].nodeid =configured_nodes->dlm_node_entries[i].dlm_node_number;
+ node_events->addrs[i].useaddr.s_addr =configured_nodes->dlm_node_entries[i].dlm_node_address.s_addr;
+ node_events->addrs[i].dlm_major= DLM_MAJOR_VERSION;
+ node_events->addrs[i].dlm_minor= DLM_MINOR_VERSION;
+
+
+ }
+
+printf("Returing the intial topology no of nodes=%d\m",configured_nodes->dlm_node_count);
+ return(configured_nodes->dlm_node_count );
+}
+#endif
diff -Nru --exclude=test dlm.old/source/include/dlm_kernel.h dlm/source/include/dlm_kernel.h
--- dlm.old/source/include/dlm_kernel.h Thu Nov 8 05:27:09 2001
+++ dlm/source/include/dlm_kernel.h Fri Mar 8 12:58:30 2002
@@ -234,12 +234,15 @@
struct timeval *t1,
struct timeval *t2);
+#ifndef LINUX_VERSION_CODE
+#include <linux/version.h>
#if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,10)
# define snprintf(buf,count,fmt, args...) \
sprintf( buf, fmt, ## args )
# define vsnprintf(buf,count,fmt,args) \
vsprintf( buf, fmt, args )
#endif
+#endif
/*
* Fake float stuff.