1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <stdio.h> 27 #include <stdlib.h> 28 #include <string.h> 29 #include <strings.h> 30 #include <sys/types.h> 31 #include <errno.h> 32 #include <syslog.h> 33 #include <unistd.h> 34 #include <sys/types.h> 35 #include <sys/socket.h> 36 #include <sys/time.h> 37 #include <netinet/in.h> 38 #include <arpa/inet.h> 39 #include <netdb.h> 40 #include <sys/stat.h> 41 #include <sys/sdt.h> 42 #include <signal.h> 43 #include <fcntl.h> 44 #include <libnvpair.h> 45 #include <libstmf.h> 46 #include <door.h> 47 #include <pthread.h> 48 #include <libscf.h> 49 #include <locale.h> 50 #include <sys/stmf_ioctl.h> 51 #include <sys/pppt_ioctl.h> 52 #include <libstmfproxy.h> 53 54 #define PPPT_NODE "/devices/pseudo/pppt@0:pppt" 55 #define USAGE "Usage: %s [-d][-f][-n nodeid] nodename\n" \ 56 "Note: nodename must be the same on both nodes\n" 57 58 59 /* 60 * static functions 61 */ 62 static void daemonInit(void); 63 static void killHandler(); 64 static int postMsg(uint_t nelem, uchar_t *aluaMsg); 65 66 67 /* 68 * globals 69 */ 70 void *t_handle; /* transport handle */ 71 char aluaNode[256]; /* one of the two alua peers */ 72 char myNode[256]; /* this hostname */ 73 int log_debug = 0; 74 int fore_ground = 0; 75 int proxy_hdl; 76 pt_ops_t *pt_ops; 77 78 /* 79 * killHandler 80 * 81 * Terminates this process on SIGQUIT, SIGINT, SIGTERM 82 */ 83 /* ARGSUSED */ 84 static void 85 killHandler(int sig) 86 { 87 exit(0); 88 } 89 90 /* 91 * doorHandler 92 * 93 * Recieve data from the local proxy port provider and relay 94 * it to the peer node. 95 */ 96 /* ARGSUSED */ 97 void 98 doorHandler( 99 void *cookie, 100 char *args, 101 size_t alen, 102 door_desc_t *ddp, 103 uint_t ndid) 104 { 105 uint32_t result = 0; 106 107 if (ddp != NULL || ndid != 0) { 108 syslog(LOG_DAEMON|LOG_WARNING, 109 "descriptor passed to door %p %d", ddp, ndid); 110 result = EINVAL; 111 } 112 113 if (args == NULL || alen == 0) { 114 syslog(LOG_DAEMON|LOG_WARNING, 115 "empty message passed to door %p %d", args, alen); 116 result = EFAULT; 117 } 118 119 if (result == 0) 120 result = postMsg((uint_t)alen, (uchar_t *)args); 121 (void) door_return((char *)&result, sizeof (result), NULL, 0); 122 123 syslog(LOG_DAEMON|LOG_WARNING, "door_return FAILED %d", errno); 124 exit(errno); 125 } 126 127 static int 128 postMsg(uint_t nelem, uchar_t *aluaMsg) 129 { 130 uint32_t buflen; 131 uchar_t *buf; 132 int ret = 0; 133 int ns; 134 135 if (t_handle == NULL) { 136 syslog(LOG_DAEMON|LOG_WARNING, 137 "postMsg() no transport handle"); 138 exit(1); 139 } 140 141 buf = malloc(nelem + sizeof (buflen)); 142 143 buflen = htonl(nelem); /* length in network byte order */ 144 bcopy(&buflen, buf, sizeof (buflen)); 145 bcopy(aluaMsg, buf + sizeof (buflen), nelem); 146 147 ns = pt_ops->stmf_proxy_send(t_handle, buf, nelem + sizeof (buflen)); 148 if (ns != nelem + sizeof (buflen)) { 149 ret = errno; 150 if (ret == 0) 151 ret = ENOTTY; /* something bogus */ 152 syslog(LOG_DAEMON|LOG_CRIT, "send() call failed: %d", ret); 153 } 154 free(buf); 155 return (ret); 156 } 157 158 /* 159 * Multi-thread the data path from the peer node to the local 160 * proxy port provider. During discover, there can be a large 161 * burst of messages from the peer node proportional to the number 162 * of LUs. Multiple threads allow these messages to be processed 163 * simultaneously. 164 */ 165 typedef struct pppt_drv_queue { 166 struct pppt_drv_queue *next; 167 uint32_t buflen; 168 uchar_t *buf; 169 } pppt_drv_queue_t; 170 171 pppt_drv_queue_t *pq_head = NULL; 172 pthread_mutex_t pq_mutex = PTHREAD_MUTEX_INITIALIZER; 173 pthread_cond_t pq_cond = PTHREAD_COND_INITIALIZER; 174 int pq_num_threads = 0; 175 int pq_avail_threads = 0; 176 177 /*ARGSUSED*/ 178 void * 179 push_to_drv(void *arg) 180 { 181 pppt_drv_queue_t *pq; 182 int rc; 183 184 (void) pthread_mutex_lock(&pq_mutex); 185 pq_num_threads++; 186 (void) pthread_mutex_unlock(&pq_mutex); 187 for (;;) { 188 (void) pthread_mutex_lock(&pq_mutex); 189 while (pq_head == NULL) { 190 pq_avail_threads++; 191 (void) pthread_cond_wait(&pq_cond, &pq_mutex); 192 pq_avail_threads--; 193 } 194 pq = pq_head; 195 pq_head = pq->next; 196 pq->next = NULL; 197 (void) pthread_mutex_unlock(&pq_mutex); 198 /* Relay the message to the local kernel */ 199 rc = stmfPostProxyMsg(proxy_hdl, (void *)pq->buf, pq->buflen); 200 if (rc != STMF_STATUS_SUCCESS) { 201 /* XXX die ? */ 202 syslog(LOG_DAEMON|LOG_CRIT, "ioctl failed - %d", errno); 203 } 204 free(pq->buf); 205 free(pq); 206 } 207 /*NOTREACHED*/ 208 return (NULL); 209 } 210 211 /* 212 * Receive data from peer and queue it up for the proxy driver. 213 */ 214 int message_count = 0; 215 static void 216 relay_peer_msg() 217 { 218 uint32_t buflen; 219 pppt_drv_queue_t *pq, *tmpq; 220 pthread_t tid; 221 int rc; 222 223 224 /* first receive the length of the message */ 225 if ((pt_ops->stmf_proxy_recv(t_handle, (uchar_t *)&buflen, 226 sizeof (buflen))) != sizeof (buflen)) { 227 syslog(LOG_DAEMON|LOG_WARNING, "recv() call failed: %d", 228 errno); 229 exit(1); 230 } 231 232 pq = malloc(sizeof (*pq)); 233 pq->next = NULL; 234 pq->buflen = ntohl(buflen); 235 pq->buf = malloc(pq->buflen+4); 236 if (log_debug) { 237 syslog(LOG_DAEMON|LOG_DEBUG, 238 "recvMsg: size of buffer - %d", (int)pq->buflen); 239 } 240 241 if ((pt_ops->stmf_proxy_recv(t_handle, pq->buf, pq->buflen)) != 242 pq->buflen) { 243 syslog(LOG_DAEMON|LOG_WARNING, "recv() call failed: %d", 244 errno); 245 exit(1); 246 } 247 248 /* Eat the first message from peer */ 249 if (message_count++ == 0) { 250 *(pq->buf+pq->buflen) = 0; 251 free(pq->buf); 252 free(pq); 253 return; 254 } 255 256 /* Queue the message to the driver */ 257 (void) pthread_mutex_lock(&pq_mutex); 258 if (pq_head == NULL) { 259 pq_head = pq; 260 } else { 261 /* add to the tail */ 262 tmpq = pq_head; 263 while (tmpq->next != NULL) 264 tmpq = tmpq->next; 265 tmpq->next = pq; 266 } 267 268 /* Make sure there is a thread to service this message */ 269 if (pq_avail_threads) { 270 /* wake an available thread */ 271 (void) pthread_cond_signal(&pq_cond); 272 (void) pthread_mutex_unlock(&pq_mutex); 273 } else { 274 /* no threads available, create a new thread */ 275 (void) pthread_mutex_unlock(&pq_mutex); 276 rc = pthread_create(&tid, NULL, push_to_drv, NULL); 277 if (rc != 0) { 278 syslog(LOG_DAEMON|LOG_WARNING, 279 "pthread_create() call failed: %d", rc); 280 if (pq_num_threads == 0) { 281 /* never created a thread */ 282 exit(rc); 283 } 284 } 285 } 286 } 287 288 /* 289 * Initialization for a daemon process 290 */ 291 static void 292 daemonInit(void) 293 { 294 pid_t pid; 295 int devnull; 296 297 if (fore_ground) 298 return; 299 300 if ((pid = fork()) < 0) { 301 syslog(LOG_DAEMON|LOG_CRIT, "Could not fork(). Exiting"); 302 exit(1); 303 } else if (pid != 0) { 304 /* 305 * XXX 306 * Simple approach for now - let the service go online. 307 * Later, set-up a pipe to the child and wait until the 308 * child indicates service is setup. 309 */ 310 exit(SMF_EXIT_OK); 311 } 312 313 (void) setsid(); 314 315 (void) chdir("/"); 316 317 (void) umask(0); 318 319 320 devnull = open("/dev/null", O_RDWR); 321 if (devnull < 0) { 322 syslog(LOG_DAEMON|LOG_CRIT, 323 "Failed to open /dev/null. Exiting"); 324 exit(1); 325 } 326 327 (void) dup2(devnull, STDIN_FILENO); 328 (void) dup2(devnull, STDOUT_FILENO); 329 (void) dup2(devnull, STDERR_FILENO); 330 (void) close(devnull); 331 } 332 333 void 334 daemon_fini(int rc) 335 { 336 /* 337 * XXX inform the parent about the service state 338 * For now, just exit on error. 339 */ 340 if (rc != 0) 341 exit(rc); 342 } 343 344 static int 345 open_proxy_driver() 346 { 347 int drv_door_fd; 348 int stmf_ret; 349 350 /* 351 * Create communication channel for the driver. 352 */ 353 if ((drv_door_fd = door_create(doorHandler, NULL, 0)) < 0) { 354 perror("door_create"); 355 syslog(LOG_DAEMON|LOG_DEBUG, 356 "could not create door: errno %d", errno); 357 return (SMF_EXIT_ERR_FATAL); 358 } 359 360 stmf_ret = stmfInitProxyDoor(&proxy_hdl, drv_door_fd); 361 if (stmf_ret != STMF_STATUS_SUCCESS) { 362 perror("pppt ioctl: door install"); 363 syslog(LOG_DAEMON|LOG_DEBUG, 364 "could not install door: errno %d", errno); 365 return (SMF_EXIT_ERR_FATAL); 366 } 367 368 return (SMF_EXIT_OK); 369 } 370 371 /* 372 * daemon entry 373 * 374 * parse arguments 375 * create resources to talk to child 376 * if !foreground 377 * daemonize, run as child 378 * open proxy driver 379 * install door in proxy driver 380 * create socket 381 * if server-side 382 * bind socket 383 * if !foreground 384 * inform parent things aok 385 * if parent 386 * exit(SMF_EXIT_OK) 387 * if server-side 388 * accept 389 * if client-side 390 * connect 391 * send hello 392 * recv hello 393 * loop on recieve 394 * XXX anyway to check in envp that we are started by SMF? 395 */ 396 int 397 main(int argc, char *argv[]) 398 { 399 struct sockaddr_in sin; 400 int rc; 401 struct sigaction act; 402 sigset_t sigmask; 403 int c; 404 int node = 0; 405 int node_override = 0; 406 int server_node = 0; 407 int server_match = 0; 408 extern char *optarg; 409 int stmf_ret; 410 411 (void) setlocale(LC_ALL, ""); 412 openlog("stmfproxy", LOG_PID, LOG_DAEMON); 413 (void) setlogmask(LOG_UPTO(LOG_INFO)); 414 415 while ((c = getopt(argc, argv, "dfn:")) != -1) { 416 switch (c) { 417 case 'd': 418 (void) setlogmask(LOG_UPTO(LOG_DEBUG)); 419 log_debug = 1; 420 break; 421 case 'f': 422 fore_ground = 1; 423 break; 424 case 'n': 425 node_override = 1; 426 node = atoi(optarg); 427 break; 428 default: 429 /* 430 * Should never happen from smf 431 */ 432 (void) fprintf(stderr, USAGE, argv[0]); 433 exit(SMF_EXIT_ERR_CONFIG); 434 break; 435 } 436 } 437 /* 438 * After the options, only the server argument should remain. 439 */ 440 if (optind != argc-1) { 441 (void) fprintf(stderr, USAGE, argv[0]); 442 exit(SMF_EXIT_ERR_CONFIG); 443 } 444 (void) strcpy(aluaNode, argv[optind]); 445 syslog(LOG_DAEMON|LOG_DEBUG, "aluaNode %s", aluaNode); 446 if (gethostname(myNode, 255)) { 447 perror("gethostname"); 448 exit(1); 449 } 450 if ((inet_aton(aluaNode, &sin.sin_addr)) == 0) { 451 /* 452 * Not ipaddr, try hostname match. 453 */ 454 server_match = (strcmp(aluaNode, myNode)) ? 0 : 1; 455 } else { 456 /* 457 * see if this is our ip address 458 */ 459 (void) fprintf(stderr, "Sorry, cannot use ip adress format\n"); 460 } 461 if (server_match) { 462 server_node = 1; 463 if (!node_override) 464 node = 1; 465 } 466 467 468 /* 469 * Allow SIGQUIT, SIGINT and SIGTERM signals to terminate us 470 */ 471 act.sa_handler = killHandler; 472 (void) sigemptyset(&act.sa_mask); 473 act.sa_flags = 0; 474 475 /* Install the signal handler */ 476 (void) sigaction(SIGQUIT, &act, NULL); 477 (void) sigaction(SIGINT, &act, NULL); 478 (void) sigaction(SIGTERM, &act, NULL); 479 (void) sigaction(SIGHUP, &act, NULL); 480 481 /* block all signals */ 482 (void) sigfillset(&sigmask); 483 484 /* unblock SIGQUIT, SIGINT, SIGTERM */ 485 (void) sigdelset(&sigmask, SIGQUIT); 486 (void) sigdelset(&sigmask, SIGINT); 487 (void) sigdelset(&sigmask, SIGTERM); 488 (void) sigdelset(&sigmask, SIGHUP); 489 490 (void) sigprocmask(SIG_SETMASK, &sigmask, NULL); 491 492 /* time to go backstage */ 493 daemonInit(); 494 495 if ((rc = open_proxy_driver()) != 0) 496 daemon_fini(rc); 497 498 if ((rc = stmf_proxy_transport_init("sockets", &pt_ops)) != 0) 499 daemon_fini(rc); 500 501 /* 502 * Establish connection 503 * 504 * At this point, the parent has exited and the service 505 * is online. But there are no real proxy services until 506 * this connect call succeeds. That could take a long time if 507 * the peer node is down. 508 */ 509 t_handle = pt_ops->stmf_proxy_connect(server_node, aluaNode); 510 if (t_handle == NULL) { 511 syslog(LOG_DAEMON|LOG_WARNING, 512 "socket() call failed: %d", errno); 513 exit(1); 514 } 515 516 /* The first message is a greeting */ 517 (void) postMsg((uint_t)strlen(myNode)+1, (uchar_t *)myNode); 518 /* Read the greeting from peer node */ 519 relay_peer_msg(); 520 /* 521 * Set the alua state in stmf. No need to keep 522 * the device open since the proxy driver has a reference. 523 */ 524 stmf_ret = stmfSetAluaState(B_TRUE, node); 525 if (stmf_ret != STMF_STATUS_SUCCESS) { 526 syslog(LOG_DAEMON|LOG_CRIT, "stmf ioctl failed - %x", stmf_ret); 527 exit(1); 528 } 529 530 /* service is online */ 531 daemon_fini(0); 532 533 /* 534 * Loop relaying data from the peer daemon to the local kernel. 535 * Data coming from the local kernel is handled asynchronously 536 * by the door server. 537 */ 538 for (;;) { /* loop forever */ 539 relay_peer_msg(); 540 } 541 } 542